From 7dada20c04975f4460abd9b99e77e39bef0b376c Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Sat, 3 Aug 2019 01:59:33 +0200 Subject: [PATCH 01/68] Upgrade to Elixir 1.8 --- mix.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index dcc1c3d..c73cb6f 100755 --- a/mix.exs +++ b/mix.exs @@ -5,7 +5,7 @@ defmodule ExDgraph.MixProject do [ app: :ex_dgraph, version: "0.2.0-beta.3", - elixir: "~> 1.6", + elixir: "~> 1.8", start_permanent: Mix.env() == :prod, deps: deps(), description: description(), From 64ab24b8c5271626f35b4fb3a3bd76ff37de3cea Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Sat, 3 Aug 2019 02:04:28 +0200 Subject: [PATCH 02/68] Update api.ex --- lib/exdgraph/api.ex | 224 ++++++++++++++++++++++++++++---------------- 1 file changed, 141 insertions(+), 83 deletions(-) diff --git a/lib/exdgraph/api.ex b/lib/exdgraph/api.ex index 37416e3..2d0d553 100755 --- a/lib/exdgraph/api.ex +++ b/lib/exdgraph/api.ex @@ -1,4 +1,6 @@ -defmodule ExDgraph.Api.Request do +alias ExDgraph.Api + +defmodule Api.Request do @moduledoc false use Protobuf, syntax: :proto3 @@ -6,17 +8,21 @@ defmodule ExDgraph.Api.Request do query: String.t(), vars: %{String.t() => String.t()}, start_ts: non_neg_integer, - lin_read: ExDgraph.Api.LinRead.t() + lin_read: Api.LinRead.t() | nil, + read_only: boolean, + best_effort: boolean } - defstruct [:query, :vars, :start_ts, :lin_read] + defstruct [:query, :vars, :start_ts, :lin_read, :read_only, :best_effort] field(:query, 1, type: :string) - field(:vars, 2, repeated: true, type: ExDgraph.Api.Request.VarsEntry, map: true) + field(:vars, 2, repeated: true, type: Api.Request.VarsEntry, map: true) field(:start_ts, 13, type: :uint64) - field(:lin_read, 14, type: ExDgraph.Api.LinRead) + field(:lin_read, 14, type: Api.LinRead) + field(:read_only, 15, type: :bool) + field(:best_effort, 16, type: :bool) end -defmodule ExDgraph.Api.Request.VarsEntry do +defmodule Api.Request.VarsEntry do @moduledoc false use Protobuf, map: true, syntax: :proto3 @@ -30,41 +36,41 @@ defmodule ExDgraph.Api.Request.VarsEntry do field(:value, 2, type: :string) end -defmodule ExDgraph.Api.Response do +defmodule Api.Response do @moduledoc false use Protobuf, syntax: :proto3 @type t :: %__MODULE__{ - json: String.t(), - schema: [ExDgraph.Api.SchemaNode.t()], - txn: ExDgraph.Api.TxnContext.t(), - latency: ExDgraph.Api.Latency.t() + json: binary, + schema: [Api.SchemaNode.t()], + txn: Api.TxnContext.t() | nil, + latency: Api.Latency.t() | nil } defstruct [:json, :schema, :txn, :latency] field(:json, 1, type: :bytes) - field(:schema, 2, repeated: true, type: ExDgraph.Api.SchemaNode) - field(:txn, 3, type: ExDgraph.Api.TxnContext) - field(:latency, 12, type: ExDgraph.Api.Latency) + field(:schema, 2, repeated: true, type: Api.SchemaNode, deprecated: true) + field(:txn, 3, type: Api.TxnContext) + field(:latency, 12, type: Api.Latency) end -defmodule ExDgraph.Api.Assigned do +defmodule Api.Assigned do @moduledoc false use Protobuf, syntax: :proto3 @type t :: %__MODULE__{ uids: %{String.t() => String.t()}, - context: ExDgraph.Api.TxnContext.t(), - latency: ExDgraph.Api.Latency.t() + context: Api.TxnContext.t() | nil, + latency: Api.Latency.t() | nil } defstruct [:uids, :context, :latency] - field(:uids, 1, repeated: true, type: ExDgraph.Api.Assigned.UidsEntry, map: true) - field(:context, 2, type: ExDgraph.Api.TxnContext) - field(:latency, 12, type: ExDgraph.Api.Latency) + field(:uids, 1, repeated: true, type: Api.Assigned.UidsEntry, map: true) + field(:context, 2, type: Api.TxnContext) + field(:latency, 12, type: Api.Latency) end -defmodule ExDgraph.Api.Assigned.UidsEntry do +defmodule Api.Assigned.UidsEntry do @moduledoc false use Protobuf, map: true, syntax: :proto3 @@ -78,17 +84,18 @@ defmodule ExDgraph.Api.Assigned.UidsEntry do field(:value, 2, type: :string) end -defmodule ExDgraph.Api.Mutation do +defmodule Api.Mutation do @moduledoc false use Protobuf, syntax: :proto3 @type t :: %__MODULE__{ - set_json: String.t(), - delete_json: String.t(), - set_nquads: String.t(), - del_nquads: String.t(), - set: [ExDgraph.Api.NQuad.t()], - del: [ExDgraph.Api.NQuad.t()], + set_json: binary, + delete_json: binary, + set_nquads: binary, + del_nquads: binary, + query: String.t(), + set: [Api.NQuad.t()], + del: [Api.NQuad.t()], start_ts: non_neg_integer, commit_now: boolean, ignore_index_conflict: boolean @@ -98,6 +105,7 @@ defmodule ExDgraph.Api.Mutation do :delete_json, :set_nquads, :del_nquads, + :query, :set, :del, :start_ts, @@ -109,56 +117,58 @@ defmodule ExDgraph.Api.Mutation do field(:delete_json, 2, type: :bytes) field(:set_nquads, 3, type: :bytes) field(:del_nquads, 4, type: :bytes) - field(:set, 10, repeated: true, type: ExDgraph.Api.NQuad) - field(:del, 11, repeated: true, type: ExDgraph.Api.NQuad) + field(:query, 5, type: :string) + field(:set, 10, repeated: true, type: Api.NQuad) + field(:del, 11, repeated: true, type: Api.NQuad) field(:start_ts, 13, type: :uint64) field(:commit_now, 14, type: :bool) field(:ignore_index_conflict, 15, type: :bool) end -defmodule ExDgraph.Api.AssignedIds do - @moduledoc false - use Protobuf, syntax: :proto3 - - @type t :: %__MODULE__{ - startId: non_neg_integer, - endId: non_neg_integer - } - defstruct [:startId, :endId] - - field(:startId, 1, type: :uint64) - field(:endId, 2, type: :uint64) -end - -defmodule ExDgraph.Api.Operation do +defmodule Api.Operation do @moduledoc false use Protobuf, syntax: :proto3 @type t :: %__MODULE__{ schema: String.t(), drop_attr: String.t(), - drop_all: boolean + drop_all: boolean, + drop_op: atom | integer, + drop_value: String.t() } - defstruct [:schema, :drop_attr, :drop_all] + defstruct [:schema, :drop_attr, :drop_all, :drop_op, :drop_value] field(:schema, 1, type: :string) field(:drop_attr, 2, type: :string) field(:drop_all, 3, type: :bool) + field(:drop_op, 4, type: Api.Operation.DropOp, enum: true) + field(:drop_value, 5, type: :string) end -defmodule ExDgraph.Api.Payload do +defmodule Api.Operation.DropOp do + @moduledoc false + use Protobuf, enum: true, syntax: :proto3 + + field(:NONE, 0) + field(:ALL, 1) + field(:DATA, 2) + field(:ATTR, 3) + field(:TYPE, 4) +end + +defmodule Api.Payload do @moduledoc false use Protobuf, syntax: :proto3 @type t :: %__MODULE__{ - Data: String.t() + Data: binary } defstruct [:Data] field(:Data, 1, type: :bytes) end -defmodule ExDgraph.Api.TxnContext do +defmodule Api.TxnContext do @moduledoc false use Protobuf, syntax: :proto3 @@ -167,25 +177,28 @@ defmodule ExDgraph.Api.TxnContext do commit_ts: non_neg_integer, aborted: boolean, keys: [String.t()], - lin_read: ExDgraph.Api.LinRead.t() + preds: [String.t()], + lin_read: Api.LinRead.t() | nil } - defstruct [:start_ts, :commit_ts, :aborted, :keys, :lin_read] + defstruct [:start_ts, :commit_ts, :aborted, :keys, :preds, :lin_read] field(:start_ts, 1, type: :uint64) field(:commit_ts, 2, type: :uint64) field(:aborted, 3, type: :bool) field(:keys, 4, repeated: true, type: :string) - field(:lin_read, 13, type: ExDgraph.Api.LinRead) + field(:preds, 5, repeated: true, type: :string) + field(:lin_read, 13, type: Api.LinRead) end -defmodule ExDgraph.Api.Check do +defmodule Api.Check do @moduledoc false use Protobuf, syntax: :proto3 + @type t :: %__MODULE__{} defstruct [] end -defmodule ExDgraph.Api.Version do +defmodule Api.Version do @moduledoc false use Protobuf, syntax: :proto3 @@ -197,19 +210,21 @@ defmodule ExDgraph.Api.Version do field(:tag, 1, type: :string) end -defmodule ExDgraph.Api.LinRead do +defmodule Api.LinRead do @moduledoc false use Protobuf, syntax: :proto3 @type t :: %__MODULE__{ - ids: %{non_neg_integer => non_neg_integer} + ids: %{non_neg_integer => non_neg_integer}, + sequencing: atom | integer } - defstruct [:ids] + defstruct [:ids, :sequencing] - field(:ids, 1, repeated: true, type: ExDgraph.Api.LinRead.IdsEntry, map: true) + field(:ids, 1, repeated: true, type: Api.LinRead.IdsEntry, map: true) + field(:sequencing, 2, type: Api.LinRead.Sequencing, enum: true) end -defmodule ExDgraph.Api.LinRead.IdsEntry do +defmodule Api.LinRead.IdsEntry do @moduledoc false use Protobuf, map: true, syntax: :proto3 @@ -223,7 +238,15 @@ defmodule ExDgraph.Api.LinRead.IdsEntry do field(:value, 2, type: :uint64) end -defmodule ExDgraph.Api.Latency do +defmodule Api.LinRead.Sequencing do + @moduledoc false + use Protobuf, enum: true, syntax: :proto3 + + field(:CLIENT_SIDE, 0) + field(:SERVER_SIDE, 1) +end + +defmodule Api.Latency do @moduledoc false use Protobuf, syntax: :proto3 @@ -239,7 +262,7 @@ defmodule ExDgraph.Api.Latency do field(:encoding_ns, 3, type: :uint64) end -defmodule ExDgraph.Api.NQuad do +defmodule Api.NQuad do @moduledoc false use Protobuf, syntax: :proto3 @@ -247,23 +270,23 @@ defmodule ExDgraph.Api.NQuad do subject: String.t(), predicate: String.t(), object_id: String.t(), - object_value: ExDgraph.Api.Value.t(), + object_value: Api.Value.t() | nil, label: String.t(), lang: String.t(), - facets: [ExDgraph.Api.Facet.t()] + facets: [Api.Facet.t()] } defstruct [:subject, :predicate, :object_id, :object_value, :label, :lang, :facets] field(:subject, 1, type: :string) field(:predicate, 2, type: :string) field(:object_id, 3, type: :string) - field(:object_value, 4, type: ExDgraph.Api.Value) + field(:object_value, 4, type: Api.Value) field(:label, 5, type: :string) field(:lang, 6, type: :string) - field(:facets, 7, repeated: true, type: ExDgraph.Api.Facet) + field(:facets, 7, repeated: true, type: Api.Facet) end -defmodule ExDgraph.Api.Value do +defmodule Api.Value do @moduledoc false use Protobuf, syntax: :proto3 @@ -286,14 +309,14 @@ defmodule ExDgraph.Api.Value do field(:uid_val, 11, type: :uint64, oneof: 0) end -defmodule ExDgraph.Api.Facet do +defmodule Api.Facet do @moduledoc false use Protobuf, syntax: :proto3 @type t :: %__MODULE__{ key: String.t(), - value: String.t(), - val_type: integer, + value: binary, + val_type: atom | integer, tokens: [String.t()], alias: String.t() } @@ -301,12 +324,12 @@ defmodule ExDgraph.Api.Facet do field(:key, 1, type: :string) field(:value, 2, type: :bytes) - field(:val_type, 3, type: ExDgraph.Api.Facet.ValType, enum: true) + field(:val_type, 3, type: Api.Facet.ValType, enum: true) field(:tokens, 4, repeated: true, type: :string) field(:alias, 5, type: :string) end -defmodule ExDgraph.Api.Facet.ValType do +defmodule Api.Facet.ValType do @moduledoc false use Protobuf, enum: true, syntax: :proto3 @@ -317,7 +340,7 @@ defmodule ExDgraph.Api.Facet.ValType do field(:DATETIME, 4) end -defmodule ExDgraph.Api.SchemaNode do +defmodule Api.SchemaNode do @moduledoc false use Protobuf, syntax: :proto3 @@ -328,9 +351,11 @@ defmodule ExDgraph.Api.SchemaNode do tokenizer: [String.t()], reverse: boolean, count: boolean, - list: boolean + list: boolean, + upsert: boolean, + lang: boolean } - defstruct [:predicate, :type, :index, :tokenizer, :reverse, :count, :list] + defstruct [:predicate, :type, :index, :tokenizer, :reverse, :count, :list, :upsert, :lang] field(:predicate, 1, type: :string) field(:type, 2, type: :string) @@ -339,20 +364,53 @@ defmodule ExDgraph.Api.SchemaNode do field(:reverse, 5, type: :bool) field(:count, 6, type: :bool) field(:list, 7, type: :bool) + field(:upsert, 8, type: :bool) + field(:lang, 9, type: :bool) +end + +defmodule Api.LoginRequest do + @moduledoc false + use Protobuf, syntax: :proto3 + + @type t :: %__MODULE__{ + userid: String.t(), + password: String.t(), + refresh_token: String.t() + } + defstruct [:userid, :password, :refresh_token] + + field(:userid, 1, type: :string) + field(:password, 2, type: :string) + field(:refresh_token, 3, type: :string) +end + +defmodule Api.Jwt do + @moduledoc false + use Protobuf, syntax: :proto3 + + @type t :: %__MODULE__{ + access_jwt: String.t(), + refresh_jwt: String.t() + } + defstruct [:access_jwt, :refresh_jwt] + + field(:access_jwt, 1, type: :string) + field(:refresh_jwt, 2, type: :string) end -defmodule ExDgraph.Api.Dgraph.Service do +defmodule Api.Dgraph.Service do @moduledoc false use GRPC.Service, name: "api.Dgraph" - rpc(:Query, ExDgraph.Api.Request, ExDgraph.Api.Response) - rpc(:Mutate, ExDgraph.Api.Mutation, ExDgraph.Api.Assigned) - rpc(:Alter, ExDgraph.Api.Operation, ExDgraph.Api.Payload) - rpc(:CommitOrAbort, ExDgraph.Api.TxnContext, ExDgraph.Api.TxnContext) - rpc(:CheckVersion, ExDgraph.Api.Check, ExDgraph.Api.Version) + rpc(:Login, Api.LoginRequest, Api.Response) + rpc(:Query, Api.Request, Api.Response) + rpc(:Mutate, Api.Mutation, Api.Assigned) + rpc(:Alter, Api.Operation, Api.Payload) + rpc(:CommitOrAbort, Api.TxnContext, Api.TxnContext) + rpc(:CheckVersion, Api.Check, Api.Version) end -defmodule ExDgraph.Api.Dgraph.Stub do +defmodule Api.Dgraph.Stub do @moduledoc false - use GRPC.Stub, service: ExDgraph.Api.Dgraph.Service + use GRPC.Stub, service: Api.Dgraph.Service end From 84ee37d8dc7c63a15d36fe0dcbb3e52db3bcee59 Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Sat, 3 Aug 2019 02:05:29 +0200 Subject: [PATCH 03/68] Upgrade to DBConnection 2. --- README.md | 7 +-- config/dev.exs | 1 - config/test.exs | 1 - lib/exdgraph.ex | 32 +++--------- lib/exdgraph/mutation.ex | 51 +++++++++---------- lib/exdgraph/mutation_statement.ex | 4 +- lib/exdgraph/operation.ex | 11 +++-- lib/exdgraph/protocol.ex | 78 +++++++++++++++++++----------- lib/exdgraph/query.ex | 8 ++- lib/exdgraph/query_statement.ex | 4 +- lib/exdgraph/utils.ex | 2 - mix.exs | 3 +- mix.lock | 2 +- test/config_test.exs | 4 +- 14 files changed, 105 insertions(+), 103 deletions(-) diff --git a/README.md b/README.md index 80d95fc..4a592d4 100755 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ **ExDgraph is functional but I would be careful using it in production. If you want to help, please drop me a message. Any help is greatly appreciated!** -ExDgraph is a gRPC based client for the [Dgraph](https://github.com/dgraph-io/dgraph) database. It uses the [DBConnection](https://hexdocs.pm/db_connection/DBConnection.html) behaviour to support transactions and connection pooling via [Poolboy](https://github.com/devinus/poolboy). Works with Dgraph v1.0.16 (latest). +ExDgraph is a gRPC based client for the [Dgraph](https://github.com/dgraph-io/dgraph) database. It uses the [DBConnection](https://hexdocs.pm/db_connection/DBConnection.html) behaviour to support transactions and connection pooling. Works with Dgraph v1.0.16 (latest). > Dgraph is an open source, horizontally scalable and distributed graph database, providing ACID transactions, consistent replication and linearizable reads. [...] Dgraph's goal is to provide Google production level scale and throughput, with low enough latency to be serving real time user queries, over terabytes of structured data. ([Source](https://github.com/dgraph-io/dgraph)) @@ -62,8 +62,7 @@ Add the configuration to your respective configuration file: config :ex_dgraph, ExDgraph, hostname: 'localhost', port: 9080, - pool_size: 5, - max_overflow: 1 + pool_size: 5 ``` And finally don't forget to add ExDgraph to the supervisor tree of your app: @@ -311,7 +310,6 @@ config :ex_dgraph, ExDgraph, # default port considered to be: 9080 hostname: 'localhost', pool_size: 5, - max_overflow: 1, ssl: true, cacertfile: '/path/to/MyRootCA.pem' ``` @@ -332,7 +330,6 @@ config :ex_dgraph, ExDgraph, # default port considered to be: 9080 hostname: 'localhost', pool_size: 5, - max_overflow: 1, ssl: true, cacertfile: '/path/to/MyRootCA.pem', certfile: '/path/to/MyClient1.pem', diff --git a/config/dev.exs b/config/dev.exs index 0a60d0b..e1e5d07 100755 --- a/config/dev.exs +++ b/config/dev.exs @@ -4,7 +4,6 @@ config :ex_dgraph, ExDgraph, # default port considered to be: 9080 hostname: 'localhost', pool_size: 5, - max_overflow: 1, # retry the request, in case of error - in the example below the retry will # linearly increase the delay from 150ms following a Fibonacci pattern, # cap the delay at 15 seconds (the value defined by the default `:timeout` diff --git a/config/test.exs b/config/test.exs index 04a88a5..d591bbb 100755 --- a/config/test.exs +++ b/config/test.exs @@ -4,7 +4,6 @@ config :ex_dgraph, ExDgraph, # default port considered to be: 9080 hostname: '0.0.0.0', pool_size: 5, - max_overflow: 1, # retry the request, in case of error - in the example below the retry will # linearly increase the delay from 150ms following a Fibonacci pattern, # cap the delay at 15 seconds (the value defined by the default `:timeout` diff --git a/lib/exdgraph.ex b/lib/exdgraph.ex index c7cee0e..6e6289d 100755 --- a/lib/exdgraph.ex +++ b/lib/exdgraph.ex @@ -1,6 +1,6 @@ defmodule ExDgraph do @moduledoc """ - ExDgraph is a gRPC based client for the Dgraph database. It uses the DBConnection behaviour to support transactions and connection pooling via Poolboy. Works with Dgraph v1.0.16 (latest). + ExDgraph is a gRPC based client for the Dgraph database. It uses DBConnection to support transactions and connection pooling. Works with Dgraph v1.0.16 (latest). ## Installation @@ -41,7 +41,6 @@ defmodule ExDgraph do hostname: 'localhost', port: 9080, pool_size: 5, - max_overflow: 1, keepalive: :infinity ``` @@ -52,9 +51,7 @@ defmodule ExDgraph do hostname: 'localhost', port: 9080, pool_size: 5, - max_overflow: 1 timeout: 15_000, # This value is used for the DBConnection timeout and the GRPC client deadline - pool: DBConnection.Poolboy, ssl: false, tls_client_auth: false, certfile: nil, @@ -309,7 +306,6 @@ defmodule ExDgraph do # default port considered to be: 9080 hostname: 'localhost', pool_size: 5, - max_overflow: 1, ssl: true, cacertfile: '/path/to/MyRootCA.pem' ``` @@ -330,7 +326,6 @@ defmodule ExDgraph do # default port considered to be: 9080 hostname: 'localhost', pool_size: 5, - max_overflow: 1, ssl: true, cacertfile: '/path/to/MyRootCA.pem', certfile: '/path/to/MyClient1.pem', @@ -366,9 +361,7 @@ defmodule ExDgraph do - `:username` - Username; - `:password` - User password; - `:pool_size` - maximum pool size; - - `:max_overflow` - maximum number of workers created if pool is empty - `:timeout` - Connect timeout in milliseconds (default: `#{@timeout}`) for DBConnection and the GRPC client deadline. - - `:pool` - The connection pool. Defaults to `DbConnection.Poolboy`. - `:ssl` - If to use ssl for the connection (please see configuration example). If you set this option, you also have to set `cacertfile` to the correct path. - `:tls_client_auth` - If to use TLS client authentication for the connection @@ -386,8 +379,7 @@ defmodule ExDgraph do config :ex_dgraph, ExDgraph, hostname: 'localhost', port: 9080, - pool_size: 5, - max_overflow: 1 + pool_size: 5 ## With SSL @@ -395,7 +387,6 @@ defmodule ExDgraph do # default port considered to be: 9080 hostname: 'localhost', pool_size: 5, - max_overflow: 1, ssl: true, cacertfile: '/path/to/MyRootCA.pem' @@ -405,7 +396,6 @@ defmodule ExDgraph do # default port considered to be: 9080 hostname: 'localhost', pool_size: 5, - max_overflow: 1, ssl: true, cacertfile: '/path/to/MyRootCA.pem', certfile: '/path/to/MyClient1.pem', @@ -423,11 +413,13 @@ defmodule ExDgraph do @doc false def init(opts) do - cnf = Utils.default_config(opts) + cnf = + Utils.default_config(opts) + |> Keyword.put(:name, :ex_dgraph_pool) children = [ {ExDgraph.ConfigAgent, cnf}, - DBConnection.child_spec(ExDgraph.Protocol, pool_config(cnf)) + DBConnection.child_spec(ExDgraph.Protocol, cnf) ] Supervisor.init(children, strategy: :one_for_one) @@ -755,16 +747,4 @@ defmodule ExDgraph do """ @spec operation!(conn, String.t()) :: ExDgraph.Response | ExDgraph.Exception defdelegate operation!(conn, statement), to: Operation - - ## Helpers - ###################### - - defp pool_config(cnf) do - [ - name: {:local, pool_name()}, - pool: Keyword.get(cnf, :pool), - pool_size: Keyword.get(cnf, :pool_size), - pool_overflow: Keyword.get(cnf, :max_overflow) - ] - end end diff --git a/lib/exdgraph/mutation.ex b/lib/exdgraph/mutation.ex index ba93702..add1d62 100644 --- a/lib/exdgraph/mutation.ex +++ b/lib/exdgraph/mutation.ex @@ -7,16 +7,19 @@ defmodule ExDgraph.Mutation do @doc false def mutation(conn, statement) do case mutation_commit(conn, statement) do - {:error, f} -> {:error, code: f.code, message: f.message} - r -> {:ok, r} + {:ok, %MutationStatement{}, result} -> + {:ok, result} + + {:error, f} -> + {:error, code: f.code, message: f.message} end end @doc false def mutation!(conn, statement) do case mutation(conn, statement) do - {:ok, r} -> - r + {:ok, %MutationStatement{}, result} -> + {:ok, result} {:error, code: code, message: message} -> raise Exception, code: code, message: message @@ -29,8 +32,11 @@ defmodule ExDgraph.Mutation do json = Poison.encode!(map_with_tmp_uids) case set_map_commit(conn, json, map_with_tmp_uids) do - {:error, f} -> {:error, code: f.code, message: f.message} - r -> {:ok, r} + {:ok, r} -> + {:ok, r} + + {:error, f} -> + {:error, code: f.code, message: f.message} end end @@ -51,8 +57,11 @@ defmodule ExDgraph.Mutation do json = Poison.encode!(uids_and_schema_map) case set_struct_commit(conn, json, uids_and_schema_map) do - {:error, f} -> {:error, code: f.code, message: f.message} - r -> {:ok, r} + {:ok, r} -> + {:ok, r} + + {:error, f} -> + {:error, code: f.code, message: f.message} end end @@ -77,8 +86,7 @@ defmodule ExDgraph.Mutation do end end - # Response.transform(DBConnection.run(conn, exec, run_opts())) - DBConnection.run(conn, exec, run_opts()) + DBConnection.run(conn, exec) end defp set_map_commit(conn, json, map_with_tmp_uids) do @@ -86,19 +94,17 @@ defmodule ExDgraph.Mutation do q = %MutationStatement{set_json: json} case DBConnection.execute(conn, q, %{}) do - {:ok, resp} -> - parsed_response = Transform.transform_mutation(resp) + {:ok, %MutationStatement{}, result} -> # Now exchange the tmp ids for the ones returned from the db - result_with_uids = replace_tmp_uids(map_with_tmp_uids, parsed_response.uids) - Map.put(parsed_response, :result, result_with_uids) + result_with_uids = replace_tmp_uids(map_with_tmp_uids, result.uids) + {:ok, Map.put(result, :result, result_with_uids)} other -> other end end - # Response.transform(DBConnection.run(conn, exec, run_opts())) - DBConnection.run(conn, exec, run_opts()) + DBConnection.run(conn, exec) end defp set_struct_commit(conn, json, struct_with_tmp_uids) do @@ -106,18 +112,17 @@ defmodule ExDgraph.Mutation do q = %MutationStatement{set_json: json} case DBConnection.execute(conn, q, %{}) do - {:ok, resp} -> - parsed_response = Transform.transform_mutation(resp) + {:ok, %MutationStatement{}, result} -> # Now exchange the tmp ids for the ones returned from the db - result_with_uids = replace_tmp_struct_uids(struct_with_tmp_uids, parsed_response.uids) - Map.put(parsed_response, :result, result_with_uids) + result_with_uids = replace_tmp_struct_uids(struct_with_tmp_uids, result.uids) + {:ok, Map.put(result, :result, result_with_uids)} other -> other end end - DBConnection.run(conn, exec, run_opts()) + DBConnection.run(conn, exec) end defp insert_tmp_uids(map) when is_list(map), do: Enum.map(map, &insert_tmp_uids/1) @@ -209,8 +214,4 @@ defmodule ExDgraph.Mutation do defp set_schema(_schema_name, {key, map_value}, result, _is_enforced_schema), do: Map.merge(result, %{key => set_tmp_ids_and_schema(map_value)}) - - defp run_opts do - [pool: ExDgraph.config(:pool)] - end end diff --git a/lib/exdgraph/mutation_statement.ex b/lib/exdgraph/mutation_statement.ex index 07aa115..1ee7235 100644 --- a/lib/exdgraph/mutation_statement.ex +++ b/lib/exdgraph/mutation_statement.ex @@ -4,11 +4,13 @@ defmodule ExDgraph.MutationStatement do end defimpl DBConnection.Query, for: ExDgraph.MutationStatement do + alias ExDgraph.Transform + def describe(query, _), do: query def parse(query, _), do: query def encode(_query, data, _), do: data - def decode(_, result, _), do: result + def decode(_query, result, _opts), do: Transform.transform_mutation(result) end diff --git a/lib/exdgraph/operation.ex b/lib/exdgraph/operation.ex index f03ce54..a69d0f7 100644 --- a/lib/exdgraph/operation.ex +++ b/lib/exdgraph/operation.ex @@ -10,8 +10,8 @@ defmodule ExDgraph.Operation do {:error, f} -> {:error, code: f.code, message: f.message} - r -> - {:ok, r} + {:ok, %OperationStatement{}, result} -> + {:ok, result} end end @@ -41,8 +41,11 @@ defmodule ExDgraph.Operation do } case DBConnection.execute(conn, operation, %{}) do - {:ok, resp} -> resp - other -> other + {:ok, resp} -> + resp + + other -> + other end end diff --git a/lib/exdgraph/protocol.ex b/lib/exdgraph/protocol.ex index 44a8ed1..906a8f1 100755 --- a/lib/exdgraph/protocol.ex +++ b/lib/exdgraph/protocol.ex @@ -13,6 +13,15 @@ defmodule ExDgraph.Protocol do alias ExDgraph.Api alias ExDgraph.{Error, Exception, MutationStatement, OperationStatement, QueryStatement} + defstruct [ + :adapter, + :channel, + :connected, + :txn_context, + :transaction_status, + txn_aborted?: false + ] + @impl true def connect(_opts) do host = to_charlist(ExDgraph.config(:hostname)) @@ -25,7 +34,8 @@ defmodule ExDgraph.Protocol do case GRPC.Stub.connect("#{host}:#{port}", opts) do {:ok, channel} -> - {:ok, channel} + state = %__MODULE__{channel: channel} + {:ok, state} {:error, reason} -> {:error, %Error{action: :connect, reason: reason}} @@ -43,27 +53,32 @@ defmodule ExDgraph.Protocol do end @impl true - def disconnect(_error, state) do - case GRPC.Stub.disconnect(state) do + def disconnect(_error, %{channel: channel} = _state) do + case GRPC.Stub.disconnect(channel) do {:ok, _} -> :ok {:error, _reason} -> :ok end end @impl true - def ping(%{adapter_payload: %{conn_pid: conn_pid}} = channel) do + def ping(%{channel: channel} = state) do + %{adapter_payload: %{conn_pid: conn_pid}} = channel # check if the server is up and wait 5s seconds before disconnect stream = :gun.head(conn_pid, "/") response = :gun.await(conn_pid, stream, 5_000) # return based on response case response do - {:response, :fin, 200, _} -> {:ok, channel} - {:error, reason} -> {:disconnect, reason, channel} + {:response, :fin, 200, _} -> {:ok, state} + {:error, reason} -> {:disconnect, reason, state} _ -> :ok end end + def handle_status(_, %{transaction_status: status} = state) do + {:idle, state} + end + @impl true def handle_begin(_opts, _state) do end @@ -77,9 +92,9 @@ defmodule ExDgraph.Protocol do end @impl true - def handle_execute(query, params, opts, channel) do + def handle_execute(query, params, opts, %{channel: channel} = state) do # only try to reconnect if the error is about the broken connection - with {:disconnect, _, _} <- execute(query, params, opts, channel) do + with {:disconnect, _, _} <- execute(query, params, opts, state) do [ delay: delay, factor: factor, @@ -95,7 +110,7 @@ defmodule ExDgraph.Protocol do retry with: delay_stream do with {:ok, channel} <- connect([]), {:ok, channel} <- checkout(channel) do - execute(query, params, opts, channel) + execute(query, params, opts, state) end after result -> result @@ -105,7 +120,7 @@ defmodule ExDgraph.Protocol do end end - @impl true + @doc false def handle_info(msg, state) do Logger.error(fn -> [inspect(__MODULE__), ?\s, inspect(self()), " received unexpected message: " | inspect(msg)] @@ -114,78 +129,83 @@ defmodule ExDgraph.Protocol do {:ok, state} end - defp execute(%QueryStatement{statement: statement}, _params, _, channel) do + defp execute( + %QueryStatement{statement: statement} = query, + _params, + _, + %{channel: channel} = state + ) do request = ExDgraph.Api.Request.new(query: statement) timeout = ExDgraph.config(:timeout) case ExDgraph.Api.Dgraph.Stub.query(channel, request, timeout: timeout) do {:ok, res} -> - {:ok, res, channel} + {:ok, query, res, state} {:error, f} -> raise Exception, code: f.status, message: f.message end rescue e -> - {:error, e, channel} + {:error, e, state} end - defp execute(%MutationStatement{statement: statement, set_json: ""}, _params, _, channel) do - request = ExDgraph.Api.Mutation.new(set_nquads: statement, commit_now: true) - do_mutate(channel, request) + defp execute(%MutationStatement{statement: statement, set_json: ""} = query, _params, _, state) do + dgraph_query = ExDgraph.Api.Mutation.new(set_nquads: statement, commit_now: true) + do_mutate(state, dgraph_query, query) end - defp execute(%MutationStatement{statement: "", set_json: set_json}, _params, _, channel) do - request = ExDgraph.Api.Mutation.new(set_json: set_json, commit_now: true) - do_mutate(channel, request) + defp execute(%MutationStatement{statement: "", set_json: set_json} = query, _params, _, state) do + dgraph_query = ExDgraph.Api.Mutation.new(set_json: set_json, commit_now: true) + do_mutate(state, dgraph_query, query) end defp execute( %MutationStatement{statement: _statement, set_json: _set_json}, _params, _, - channel + %{channel: channel} = state ) do raise Exception, code: 2, message: "Both set_json and statement defined" rescue e -> - {:error, e, channel} + {:error, e, state} end defp execute( - %OperationStatement{drop_all: drop_all, schema: schema, drop_attr: drop_attr}, + %OperationStatement{drop_all: drop_all, schema: schema, drop_attr: drop_attr} = query, _params, _, - channel + %{channel: channel} = state ) do operation = Api.Operation.new(drop_all: drop_all, schema: schema, drop_attr: drop_attr) timeout = ExDgraph.config(:timeout) case ExDgraph.Api.Dgraph.Stub.alter(channel, operation, timeout: timeout) do {:ok, res} -> - {:ok, res, channel} + {:ok, query, res, state} {:error, f} -> raise Exception, code: f.status, message: f.message end rescue e -> - {:error, e, channel} + {:error, e, state} end - defp do_mutate(channel, request) do + defp do_mutate(%{channel: channel} = state, dgraph_query, query) do timeout = ExDgraph.config(:timeout) - case ExDgraph.Api.Dgraph.Stub.mutate(channel, request, timeout: timeout) do + case ExDgraph.Api.Dgraph.Stub.mutate(channel, dgraph_query, timeout: timeout) do {:ok, res} -> - {:ok, res, channel} + {:ok, query, res, state} {:error, f} -> raise Exception, code: f.status, message: f.message end rescue e -> - {:error, e, channel} + {:error, e, state} end defp configure_ssl(ssl_opts \\ []) do diff --git a/lib/exdgraph/query.ex b/lib/exdgraph/query.ex index 5b00268..545b668 100644 --- a/lib/exdgraph/query.ex +++ b/lib/exdgraph/query.ex @@ -7,8 +7,11 @@ defmodule ExDgraph.Query do @doc false def query(conn, statement) do case query_commit(conn, statement) do - {:error, f} -> {:error, code: Map.get(f, :code, Map.get(f, :status)), message: f.message} - r -> {:ok, r} + {:error, f} -> + {:error, code: Map.get(f, :code, Map.get(f, :status)), message: f.message} + + {:ok, %QueryStatement{}, result} -> + {:ok, result} end end @@ -32,6 +35,7 @@ defmodule ExDgraph.Query do other -> other end end + DBConnection.run(conn, exec, run_opts()) end diff --git a/lib/exdgraph/query_statement.ex b/lib/exdgraph/query_statement.ex index 2b63b21..5af8a32 100644 --- a/lib/exdgraph/query_statement.ex +++ b/lib/exdgraph/query_statement.ex @@ -4,11 +4,13 @@ defmodule ExDgraph.QueryStatement do end defimpl DBConnection.Query, for: ExDgraph.QueryStatement do + alias ExDgraph.Transform + def describe(query, _), do: query def parse(query, _), do: query def encode(_query, data, _), do: data - def decode(_, result, _), do: result + def decode(_query, result, _opts), do: Transform.transform_query(result) end diff --git a/lib/exdgraph/utils.ex b/lib/exdgraph/utils.ex index 77663e7..d3b8b4d 100755 --- a/lib/exdgraph/utils.ex +++ b/lib/exdgraph/utils.ex @@ -92,9 +92,7 @@ defmodule ExDgraph.Utils do |> Keyword.put_new(:hostname, System.get_env("DGRAPH_HOST") || 'localhost') |> Keyword.put_new(:port, System.get_env("DGRAPH_PORT") || 9080) |> Keyword.put_new(:pool_size, 5) - |> Keyword.put_new(:max_overflow, 2) |> Keyword.put_new(:timeout, 15_000) - |> Keyword.put_new(:pool, DBConnection.Poolboy) |> Keyword.put_new(:ssl, false) |> Keyword.put_new(:tls_client_auth, false) |> Keyword.put_new(:certfile, nil) diff --git a/mix.exs b/mix.exs index c73cb6f..3b2e589 100755 --- a/mix.exs +++ b/mix.exs @@ -45,8 +45,7 @@ defmodule ExDgraph.MixProject do {:grpc, "~> 0.3.1"}, {:protobuf, "~> 0.6.1"}, {:poison, "~> 3.1"}, - {:poolboy, "~> 1.5.2"}, - {:db_connection, "~> 1.1"}, + {:db_connection, "~> 2.1"}, {:retry, "~> 0.11.2"}, {:morphix, "~> 0.6.0"}, {:ex_doc, "~> 0.19.0", only: :dev, runtime: false}, diff --git a/mix.lock b/mix.lock index cabd573..3166b36 100755 --- a/mix.lock +++ b/mix.lock @@ -6,7 +6,7 @@ "cowboy": {:hex, :cowboy, "2.5.0", "4ef3ae066ee10fe01ea3272edc8f024347a0d3eb95f6fbb9aed556dacbfc1337", [:rebar3], [{:cowlib, "~> 2.6.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "~> 1.6.2", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm"}, "cowlib": {:hex, :cowlib, "2.6.0", "8aa629f81a0fc189f261dc98a42243fa842625feea3c7ec56c48f4ccdb55490f", [:rebar3], [], "hexpm"}, "credo": {:hex, :credo, "1.0.2", "88bc918f215168bf6ce7070610a6173c45c82f32baa08bdfc80bf58df2d103b6", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, - "db_connection": {:hex, :db_connection, "1.1.3", "89b30ca1ef0a3b469b1c779579590688561d586694a3ce8792985d4d7e575a61", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}, {:poolboy, "~> 1.5", [hex: :poolboy, repo: "hexpm", optional: true]}, {:sbroker, "~> 1.0", [hex: :sbroker, repo: "hexpm", optional: true]}], "hexpm"}, + "db_connection": {:hex, :db_connection, "2.1.1", "a51e8a2ee54ef2ae6ec41a668c85787ed40cb8944928c191280fe34c15b76ae5", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm"}, "earmark": {:hex, :earmark, "1.3.4", "52aba89c60529284df5fc18adc4c808b7346e72668bc2fb2b68d7394996c4af8", [:mix], [], "hexpm"}, "elixir_uuid": {:hex, :elixir_uuid, "1.2.0", "ff26e938f95830b1db152cb6e594d711c10c02c6391236900ddd070a6b01271d", [:mix], [], "hexpm"}, "ex_doc": {:hex, :ex_doc, "0.19.3", "3c7b0f02851f5fc13b040e8e925051452e41248f685e40250d7e40b07b9f8c10", [:mix], [{:earmark, "~> 1.2", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.10", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm"}, diff --git a/test/config_test.exs b/test/config_test.exs index 3d27362..f77698e 100755 --- a/test/config_test.exs +++ b/test/config_test.exs @@ -5,8 +5,7 @@ defmodule Config.Test do @basic_config [ hostname: 'ole', port: 1234, - pool_size: 10, - max_overflow: 7 + pool_size: 10 ] @ssl_config [ @@ -20,7 +19,6 @@ defmodule Config.Test do assert config[:hostname] == 'ole' assert config[:port] == 1234 assert config[:pool_size] == 10 - assert config[:max_overflow] == 7 assert config[:ssl] == false assert config[:tls_client_auth] == false assert config[:enforce_struct_schema] == false From 629c6abaa896244abde4a8392c89b13c607e9ef0 Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Sat, 3 Aug 2019 02:24:13 +0200 Subject: [PATCH 04/68] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4a592d4..857909b 100755 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ **ExDgraph is functional but I would be careful using it in production. If you want to help, please drop me a message. Any help is greatly appreciated!** -ExDgraph is a gRPC based client for the [Dgraph](https://github.com/dgraph-io/dgraph) database. It uses the [DBConnection](https://hexdocs.pm/db_connection/DBConnection.html) behaviour to support transactions and connection pooling. Works with Dgraph v1.0.16 (latest). +ExDgraph is a gRPC based client for the [Dgraph](https://github.com/dgraph-io/dgraph) database. It uses [DBConnection](https://hexdocs.pm/db_connection/DBConnection.html) to support transactions and connection pooling. Works with Dgraph v1.0.16 (latest). > Dgraph is an open source, horizontally scalable and distributed graph database, providing ACID transactions, consistent replication and linearizable reads. [...] Dgraph's goal is to provide Google production level scale and throughput, with low enough latency to be serving real time user queries, over terabytes of structured data. ([Source](https://github.com/dgraph-io/dgraph)) From e35f5099ececdb0e7b97cbb14f3831139d932a09 Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Wed, 7 Aug 2019 09:35:58 +0200 Subject: [PATCH 05/68] Switch from Poison to Jason. --- README.md | 2 +- lib/exdgraph.ex | 2 +- lib/exdgraph/mutation.ex | 4 ++-- lib/exdgraph/transform.ex | 2 +- lib/exdgraph/utils.ex | 2 +- mix.exs | 14 +++++++------- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 857909b..40a77ea 100755 --- a/README.md +++ b/README.md @@ -298,7 +298,7 @@ request = ExDgraph.Api.Request.new(query: query) {:ok, msg} = channel |> ExDgraph.Api.Dgraph.Stub.query(request) # Parse result -json = Poison.decode!(msg.json) +json = Jason.decode!(msg.json) ``` ## Using SSL diff --git a/lib/exdgraph.ex b/lib/exdgraph.ex index 6e6289d..41fc3aa 100755 --- a/lib/exdgraph.ex +++ b/lib/exdgraph.ex @@ -294,7 +294,7 @@ defmodule ExDgraph do {:ok, msg} = channel |> ExDgraph.Api.Dgraph.Stub.query(request) # Parse result - json = Poison.decode!(msg.json) + json = Jason.decode!(msg.json) ``` ## Using SSL diff --git a/lib/exdgraph/mutation.ex b/lib/exdgraph/mutation.ex index add1d62..2676e3c 100644 --- a/lib/exdgraph/mutation.ex +++ b/lib/exdgraph/mutation.ex @@ -29,7 +29,7 @@ defmodule ExDgraph.Mutation do @doc false def set_map(conn, map) do map_with_tmp_uids = insert_tmp_uids(map) - json = Poison.encode!(map_with_tmp_uids) + json = Jason.encode!(map_with_tmp_uids) case set_map_commit(conn, json, map_with_tmp_uids) do {:ok, r} -> @@ -54,7 +54,7 @@ defmodule ExDgraph.Mutation do @doc false def set_struct(conn, struct) do uids_and_schema_map = set_tmp_ids_and_schema(struct) - json = Poison.encode!(uids_and_schema_map) + json = Jason.encode!(uids_and_schema_map) case set_struct_commit(conn, json, uids_and_schema_map) do {:ok, r} -> diff --git a/lib/exdgraph/transform.ex b/lib/exdgraph/transform.ex index bb98051..59b3e4a 100644 --- a/lib/exdgraph/transform.ex +++ b/lib/exdgraph/transform.ex @@ -8,7 +8,7 @@ defmodule ExDgraph.Transform do keys into atom keys. """ def transform_query(%ExDgraph.Api.Response{json: json, schema: schema, txn: txn}) do - decoded = Poison.decode!(json) + decoded = Jason.decode!(json) transformed = case Morphix.atomorphiform(decoded) do diff --git a/lib/exdgraph/utils.ex b/lib/exdgraph/utils.ex index d3b8b4d..f3ed7cb 100755 --- a/lib/exdgraph/utils.ex +++ b/lib/exdgraph/utils.ex @@ -5,7 +5,7 @@ defmodule ExDgraph.Utils do def as_rendered(value) do case value do - x when is_list(x) -> x |> Poison.encode!() + x when is_list(x) -> x |> Jason.encode!() %Date{} = x -> x |> Date.to_iso8601() |> Kernel.<>("T00:00:00.0+00:00") %DateTime{} = x -> x |> DateTime.to_iso8601() |> String.replace("Z", "+00:00") x -> x |> to_string diff --git a/mix.exs b/mix.exs index 3b2e589..4c202de 100755 --- a/mix.exs +++ b/mix.exs @@ -42,17 +42,17 @@ defmodule ExDgraph.MixProject do # Run "mix help deps" to learn about dependencies. defp deps do [ + {:db_connection, "~> 2.1"}, + {:elixir_uuid, "~> 1.2"}, {:grpc, "~> 0.3.1"}, + {:jason, "~> 1.1"}, + {:morphix, "~> 0.6.0"}, {:protobuf, "~> 0.6.1"}, - {:poison, "~> 3.1"}, - {:db_connection, "~> 2.1"}, {:retry, "~> 0.11.2"}, - {:morphix, "~> 0.6.0"}, - {:ex_doc, "~> 0.19.0", only: :dev, runtime: false}, - {:elixir_uuid, "~> 1.2"}, - {:mix_test_watch, "~> 0.5", only: :dev, runtime: false}, {:excoveralls, "~> 0.10", only: :test}, - {:credo, "~> 1.0", only: [:dev, :test], runtime: false} + {:credo, "~> 1.0", only: [:dev, :test], runtime: false}, + {:ex_doc, "~> 0.19.0", only: :dev, runtime: false}, + {:mix_test_watch, "~> 0.5", only: :dev, runtime: false} ] end From dfe7b5315dedb2b5c0bcb1c6db00d438f945b043 Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Wed, 7 Aug 2019 10:26:21 +0200 Subject: [PATCH 06/68] Start moving connection related functionality to an Adapter module. --- lib/exdgraph/adapter.ex | 16 ++++++++++++++++ lib/exdgraph/protocol.ex | 15 ++++++++++----- 2 files changed, 26 insertions(+), 5 deletions(-) create mode 100644 lib/exdgraph/adapter.ex diff --git a/lib/exdgraph/adapter.ex b/lib/exdgraph/adapter.ex new file mode 100644 index 0000000..8f2fff8 --- /dev/null +++ b/lib/exdgraph/adapter.ex @@ -0,0 +1,16 @@ +defmodule ExDgraph.Adapter do + alias ExDgraph.Api.Dgraph.Stub, as: ApiStub + alias ExDgraph.Error + + require Logger + + def connect(host, port, opts \\ []) do + case GRPC.Stub.connect("#{host}:#{port}", opts) do + {:ok, channel} -> + {:ok, channel} + + {:error, reason} -> + {:error, %Error{action: :connect, reason: reason}} + end + end +end diff --git a/lib/exdgraph/protocol.ex b/lib/exdgraph/protocol.ex index 906a8f1..d6a30fc 100755 --- a/lib/exdgraph/protocol.ex +++ b/lib/exdgraph/protocol.ex @@ -1,8 +1,6 @@ defmodule ExDgraph.Protocol do @moduledoc """ Implements callbacks required by DBConnection. - - Each callback receives an open connection as a state. """ use DBConnection @@ -10,8 +8,15 @@ defmodule ExDgraph.Protocol do require Logger - alias ExDgraph.Api - alias ExDgraph.{Error, Exception, MutationStatement, OperationStatement, QueryStatement} + alias ExDgraph.{ + Adapter, + Api, + Error, + Exception, + MutationStatement, + OperationStatement, + QueryStatement + } defstruct [ :adapter, @@ -32,7 +37,7 @@ defmodule ExDgraph.Protocol do |> set_ssl_opts() |> Keyword.put(:adapter_opts, %{http2_opts: %{keepalive: ExDgraph.config(:keepalive)}}) - case GRPC.Stub.connect("#{host}:#{port}", opts) do + case Adapter.connect(host, port, opts) do {:ok, channel} -> state = %__MODULE__{channel: channel} {:ok, state} From e3361d8349da996e186db42f971a1e001ebc10d8 Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Wed, 7 Aug 2019 10:31:49 +0200 Subject: [PATCH 07/68] Require at least Elixir 1.7. and also test against 1.9. --- .travis.yml | 6 +++++- mix.exs | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index f044c20..6e9db8d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,9 @@ language: elixir elixir: - - 1.6 + - 1.7 - 1.8 + - 1.9 otp_release: - 20.0 @@ -11,6 +12,9 @@ matrix: include: - elixir: 1.8 otp_release: 21.2 + - elixir: 1.9 + otp_release: + otp_release: 22.0 cache: directories: diff --git a/mix.exs b/mix.exs index 4c202de..dbeb5aa 100755 --- a/mix.exs +++ b/mix.exs @@ -5,7 +5,7 @@ defmodule ExDgraph.MixProject do [ app: :ex_dgraph, version: "0.2.0-beta.3", - elixir: "~> 1.8", + elixir: "~> 1.7", start_permanent: Mix.env() == :prod, deps: deps(), description: description(), From d7f3caab7e75a6cbf765df5426a9529a8b1cc20a Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Wed, 7 Aug 2019 16:55:32 +0200 Subject: [PATCH 08/68] Remove adapter. --- lib/exdgraph/adapter.ex | 16 ---------------- lib/exdgraph/protocol.ex | 3 +-- 2 files changed, 1 insertion(+), 18 deletions(-) delete mode 100644 lib/exdgraph/adapter.ex diff --git a/lib/exdgraph/adapter.ex b/lib/exdgraph/adapter.ex deleted file mode 100644 index 8f2fff8..0000000 --- a/lib/exdgraph/adapter.ex +++ /dev/null @@ -1,16 +0,0 @@ -defmodule ExDgraph.Adapter do - alias ExDgraph.Api.Dgraph.Stub, as: ApiStub - alias ExDgraph.Error - - require Logger - - def connect(host, port, opts \\ []) do - case GRPC.Stub.connect("#{host}:#{port}", opts) do - {:ok, channel} -> - {:ok, channel} - - {:error, reason} -> - {:error, %Error{action: :connect, reason: reason}} - end - end -end diff --git a/lib/exdgraph/protocol.ex b/lib/exdgraph/protocol.ex index d6a30fc..e8a97be 100755 --- a/lib/exdgraph/protocol.ex +++ b/lib/exdgraph/protocol.ex @@ -9,7 +9,6 @@ defmodule ExDgraph.Protocol do require Logger alias ExDgraph.{ - Adapter, Api, Error, Exception, @@ -37,7 +36,7 @@ defmodule ExDgraph.Protocol do |> set_ssl_opts() |> Keyword.put(:adapter_opts, %{http2_opts: %{keepalive: ExDgraph.config(:keepalive)}}) - case Adapter.connect(host, port, opts) do + case GRPC.Stub.connect("#{host}:#{port}", opts) do {:ok, channel} -> state = %__MODULE__{channel: channel} {:ok, state} From 8c70785df34893d0ae73a1dc02318acd449aa61e Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Wed, 7 Aug 2019 17:53:18 +0200 Subject: [PATCH 09/68] Remove Retry since it is handled by DBConnection 2. --- Changelog.md | 2 + config/dev.exs | 8 ---- config/test.exs | 8 ---- lib/exdgraph.ex | 6 +-- lib/exdgraph/protocol.ex | 86 ++++++++++++++++---------------------- lib/exdgraph/utils.ex | 3 +- mix.exs | 2 - mix.lock | 2 +- test/retry_backof_test.exs | 31 -------------- 9 files changed, 41 insertions(+), 107 deletions(-) delete mode 100644 test/retry_backof_test.exs diff --git a/Changelog.md b/Changelog.md index 3e0a2de..3969adf 100644 --- a/Changelog.md +++ b/Changelog.md @@ -6,6 +6,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] +- Remove retry since DBConnection 2 implements a retry mechanism. + ## [0.2.0-beta.3] - 2019-07-31 - Test against Dgraph `v1.0.16` diff --git a/config/dev.exs b/config/dev.exs index e1e5d07..a9cf331 100755 --- a/config/dev.exs +++ b/config/dev.exs @@ -4,12 +4,4 @@ config :ex_dgraph, ExDgraph, # default port considered to be: 9080 hostname: 'localhost', pool_size: 5, - # retry the request, in case of error - in the example below the retry will - # linearly increase the delay from 150ms following a Fibonacci pattern, - # cap the delay at 15 seconds (the value defined by the default `:timeout` - # parameter) and giving up after 3 attempts - retry_linear_backoff: [delay: 150, factor: 2, tries: 3], enforce_struct_schema: false - -# the `retry_linear_backoff` values above are also the default driver values, -# re-defined here mostly as a reminder diff --git a/config/test.exs b/config/test.exs index d591bbb..9cbb857 100755 --- a/config/test.exs +++ b/config/test.exs @@ -4,12 +4,4 @@ config :ex_dgraph, ExDgraph, # default port considered to be: 9080 hostname: '0.0.0.0', pool_size: 5, - # retry the request, in case of error - in the example below the retry will - # linearly increase the delay from 150ms following a Fibonacci pattern, - # cap the delay at 15 seconds (the value defined by the default `:timeout` - # parameter) and giving up after 3 attempts - retry_linear_backoff: [delay: 150, factor: 2, tries: 3], enforce_struct_schema: true - -# the `retry_linear_backoff` values above are also the default driver values, -# re-defined here mostly as a reminder diff --git a/lib/exdgraph.ex b/lib/exdgraph.ex index 41fc3aa..619e69d 100755 --- a/lib/exdgraph.ex +++ b/lib/exdgraph.ex @@ -343,7 +343,7 @@ defmodule ExDgraph do use Supervisor - @pool_name :ex_dgraph_pool + @pool_name :ex_dgraph @timeout 15_000 alias ExDgraph.Api.{Mutation, Operation} @@ -413,9 +413,7 @@ defmodule ExDgraph do @doc false def init(opts) do - cnf = - Utils.default_config(opts) - |> Keyword.put(:name, :ex_dgraph_pool) + cnf = Utils.default_config(opts) children = [ {ExDgraph.ConfigAgent, cnf}, diff --git a/lib/exdgraph/protocol.ex b/lib/exdgraph/protocol.ex index e8a97be..4e168c3 100755 --- a/lib/exdgraph/protocol.ex +++ b/lib/exdgraph/protocol.ex @@ -4,7 +4,6 @@ defmodule ExDgraph.Protocol do """ use DBConnection - use Retry require Logger @@ -18,7 +17,6 @@ defmodule ExDgraph.Protocol do } defstruct [ - :adapter, :channel, :connected, :txn_context, @@ -95,35 +93,6 @@ defmodule ExDgraph.Protocol do def handle_commit(_opts, _state) do end - @impl true - def handle_execute(query, params, opts, %{channel: channel} = state) do - # only try to reconnect if the error is about the broken connection - with {:disconnect, _, _} <- execute(query, params, opts, state) do - [ - delay: delay, - factor: factor, - tries: tries - ] = ExDgraph.config(:retry_linear_backoff) - - delay_stream = - delay - |> linear_backoff(factor) - |> cap(ExDgraph.config(:timeout)) - |> Stream.take(tries) - - retry with: delay_stream do - with {:ok, channel} <- connect([]), - {:ok, channel} <- checkout(channel) do - execute(query, params, opts, state) - end - after - result -> result - else - error -> error - end - end - end - @doc false def handle_info(msg, state) do Logger.error(fn -> @@ -133,12 +102,13 @@ defmodule ExDgraph.Protocol do {:ok, state} end - defp execute( - %QueryStatement{statement: statement} = query, - _params, - _, - %{channel: channel} = state - ) do + @impl true + def handle_execute( + %QueryStatement{statement: statement} = query, + _params, + _, + %{channel: channel} = state + ) do request = ExDgraph.Api.Request.new(query: statement) timeout = ExDgraph.config(:timeout) @@ -154,34 +124,48 @@ defmodule ExDgraph.Protocol do {:error, e, state} end - defp execute(%MutationStatement{statement: statement, set_json: ""} = query, _params, _, state) do + @impl true + def handle_execute( + %MutationStatement{statement: statement, set_json: ""} = query, + _params, + _, + state + ) do dgraph_query = ExDgraph.Api.Mutation.new(set_nquads: statement, commit_now: true) do_mutate(state, dgraph_query, query) end - defp execute(%MutationStatement{statement: "", set_json: set_json} = query, _params, _, state) do + @impl true + def handle_execute( + %MutationStatement{statement: "", set_json: set_json} = query, + _params, + _, + state + ) do dgraph_query = ExDgraph.Api.Mutation.new(set_json: set_json, commit_now: true) do_mutate(state, dgraph_query, query) end - defp execute( - %MutationStatement{statement: _statement, set_json: _set_json}, - _params, - _, - %{channel: channel} = state - ) do + @impl true + def handle_execute( + %MutationStatement{statement: _statement, set_json: _set_json}, + _params, + _, + %{channel: channel} = state + ) do raise Exception, code: 2, message: "Both set_json and statement defined" rescue e -> {:error, e, state} end - defp execute( - %OperationStatement{drop_all: drop_all, schema: schema, drop_attr: drop_attr} = query, - _params, - _, - %{channel: channel} = state - ) do + @impl true + def handle_execute( + %OperationStatement{drop_all: drop_all, schema: schema, drop_attr: drop_attr} = query, + _params, + _, + %{channel: channel} = state + ) do operation = Api.Operation.new(drop_all: drop_all, schema: schema, drop_attr: drop_attr) timeout = ExDgraph.config(:timeout) diff --git a/lib/exdgraph/utils.ex b/lib/exdgraph/utils.ex index f3ed7cb..2e7e8f0 100755 --- a/lib/exdgraph/utils.ex +++ b/lib/exdgraph/utils.ex @@ -91,14 +91,13 @@ defmodule ExDgraph.Utils do config |> Keyword.put_new(:hostname, System.get_env("DGRAPH_HOST") || 'localhost') |> Keyword.put_new(:port, System.get_env("DGRAPH_PORT") || 9080) - |> Keyword.put_new(:pool_size, 5) + |> Keyword.put_new(:name, :ex_dgraph) |> Keyword.put_new(:timeout, 15_000) |> Keyword.put_new(:ssl, false) |> Keyword.put_new(:tls_client_auth, false) |> Keyword.put_new(:certfile, nil) |> Keyword.put_new(:keyfile, nil) |> Keyword.put_new(:cacertfile, nil) - |> Keyword.put_new(:retry_linear_backoff, delay: 150, factor: 2, tries: 3) |> Keyword.put_new(:enforce_struct_schema, false) |> Keyword.put_new(:keepalive, :infinity) |> Enum.reject(fn {_k, v} -> is_nil(v) end) diff --git a/mix.exs b/mix.exs index dbeb5aa..3302a48 100755 --- a/mix.exs +++ b/mix.exs @@ -33,7 +33,6 @@ defmodule ExDgraph.MixProject do applications: [ :logger, :db_connection, - :retry, :grpc ] ] @@ -48,7 +47,6 @@ defmodule ExDgraph.MixProject do {:jason, "~> 1.1"}, {:morphix, "~> 0.6.0"}, {:protobuf, "~> 0.6.1"}, - {:retry, "~> 0.11.2"}, {:excoveralls, "~> 0.10", only: :test}, {:credo, "~> 1.0", only: [:dev, :test], runtime: false}, {:ex_doc, "~> 0.19.0", only: :dev, runtime: false}, diff --git a/mix.lock b/mix.lock index 3166b36..e5f1d12 100755 --- a/mix.lock +++ b/mix.lock @@ -33,7 +33,7 @@ "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm"}, "protobuf": {:hex, :protobuf, "0.6.1", "3ae3ca63d12d1a8c5b10c2e5fbc5145dccbff842ccd541109b74b20b631cea3b", [:mix], [], "hexpm"}, "ranch": {:hex, :ranch, "1.6.2", "6db93c78f411ee033dbb18ba8234c5574883acb9a75af0fb90a9b82ea46afa00", [:rebar3], [], "hexpm"}, - "retry": {:hex, :retry, "0.11.2", "29f9ab8e7d78878307f4653adc286d8a8baa6b66b6bcb67925c07a1386ef7867", [:mix], [], "hexpm"}, + "retry": {:hex, :retry, "0.13.0", "bb9b2713f70f39337837852337ad280c77662574f4fb852a8386c269f3d734c4", [:mix], [], "hexpm"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"}, "unicode_util_compat": {:hex, :unicode_util_compat, "0.4.1", "d869e4c68901dd9531385bb0c8c40444ebf624e60b6962d95952775cac5e90cd", [:rebar3], [], "hexpm"}, "uuid": {:hex, :uuid, "1.1.8", "e22fc04499de0de3ed1116b770c7737779f226ceefa0badb3592e64d5cfb4eb9", [:mix], [], "hexpm"}, diff --git a/test/retry_backof_test.exs b/test/retry_backof_test.exs deleted file mode 100644 index 7f34ddd..0000000 --- a/test/retry_backof_test.exs +++ /dev/null @@ -1,31 +0,0 @@ -defmodule Retry.Backoff.Test do - use ExUnit.Case, async: true - import Stream - - use Retry - - setup_all do - {:ok, [conn: ExDgraph.conn()]} - end - - test "retry retries execution for specified attempts using an invalid GraphQL+ query", - context do - conn = context[:conn] - - {elapsed, _} = - :timer.tc(fn -> - {:error, [code: 2, message: message]} = - retry with: 500 |> linear_backoff(1) |> take(5) do - ExDgraph.query(conn, "INVALID") - after - result -> result - else - error -> error - end - - assert message =~ "while lexing INVALID: Invalid operation type: INVALID" - end) - - assert elapsed / 1000 >= 2500 - end -end From 5486b5da903e3d8095e00931479fb7bf620f1f39 Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Wed, 7 Aug 2019 17:53:27 +0200 Subject: [PATCH 10/68] Add DBConnection config options. --- lib/exdgraph/utils.ex | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/exdgraph/utils.ex b/lib/exdgraph/utils.ex index 2e7e8f0..8f8839a 100755 --- a/lib/exdgraph/utils.ex +++ b/lib/exdgraph/utils.ex @@ -100,6 +100,15 @@ defmodule ExDgraph.Utils do |> Keyword.put_new(:cacertfile, nil) |> Keyword.put_new(:enforce_struct_schema, false) |> Keyword.put_new(:keepalive, :infinity) + # DBConnection config options + |> Keyword.put_new(:name, 1_000) + |> Keyword.put_new(:backoff_min, 1_000) + |> Keyword.put_new(:backoff_max, 30_000) + |> Keyword.put_new(:backoff_type, :rand_exp) + |> Keyword.put_new(:pool_size, 5) + |> Keyword.put_new(:idle_interval, 5_000) + |> Keyword.put_new(:max_restarts, 3) + |> Keyword.put_new(:max_seconds, 5) |> Enum.reject(fn {_k, v} -> is_nil(v) end) end end From b578931df2c65e9a800a89cf86b7ed8cccb6ab85 Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Tue, 3 Sep 2019 00:15:58 +0200 Subject: [PATCH 11/68] Delete config_test.exs --- test/config_test.exs | 45 -------------------------------------------- 1 file changed, 45 deletions(-) delete mode 100755 test/config_test.exs diff --git a/test/config_test.exs b/test/config_test.exs deleted file mode 100755 index f77698e..0000000 --- a/test/config_test.exs +++ /dev/null @@ -1,45 +0,0 @@ -defmodule Config.Test do - use ExUnit.Case, async: true - alias ExDgraph.Utils - - @basic_config [ - hostname: 'ole', - port: 1234, - pool_size: 10 - ] - - @ssl_config [ - ssl: true, - cacertfile: "MyRootCA.pem" - ] - - test "standard ExDgraph configuration parameters" do - config = Utils.default_config(@basic_config) - - assert config[:hostname] == 'ole' - assert config[:port] == 1234 - assert config[:pool_size] == 10 - assert config[:ssl] == false - assert config[:tls_client_auth] == false - assert config[:enforce_struct_schema] == false - assert config[:keepalive] == :infinity - end - - test "standard ExDgraph default configuration" do - config = Utils.default_config([]) - - assert config[:hostname] == 'localhost' - assert config[:port] == 9080 - assert config[:ssl] == false - assert config[:tls_client_auth] == false - assert config[:enforce_struct_schema] == false - assert config[:keepalive] == :infinity - end - - test "ssl config from parameters" do - config = Utils.default_config(@ssl_config) - - assert config[:ssl] == true - assert config[:cacertfile] == "MyRootCA.pem" - end -end From f50db88f249b0d9d9c7d941693c1682e3430845b Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Tue, 3 Sep 2019 00:16:14 +0200 Subject: [PATCH 12/68] Update test_helper.exs --- test/test_helper.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_helper.exs b/test/test_helper.exs index 8fec030..5573444 100755 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -80,4 +80,4 @@ if Process.whereis(ExDgraph.pool_name()) == nil do {:ok, _pid} = ExDgraph.start_link(Application.get_env(:ex_dgraph, ExDgraph)) end -Process.flag(:trap_exit, true) +ExUnit.configure(exclude: [tls_tests: true]) From ded0ea3d6cad59639346872c055475b263de3d7b Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Tue, 3 Sep 2019 00:16:39 +0200 Subject: [PATCH 13/68] Always use latest version of ex_doc. --- mix.exs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/mix.exs b/mix.exs index 3302a48..9f768c5 100755 --- a/mix.exs +++ b/mix.exs @@ -30,11 +30,7 @@ defmodule ExDgraph.MixProject do # Run "mix help compile.app" to learn about applications. def application do [ - applications: [ - :logger, - :db_connection, - :grpc - ] + applications: [:logger, :db_connection, :grpc] ] end @@ -49,7 +45,7 @@ defmodule ExDgraph.MixProject do {:protobuf, "~> 0.6.1"}, {:excoveralls, "~> 0.10", only: :test}, {:credo, "~> 1.0", only: [:dev, :test], runtime: false}, - {:ex_doc, "~> 0.19.0", only: :dev, runtime: false}, + {:ex_doc, "> 0.0.0", only: :dev, runtime: false}, {:mix_test_watch, "~> 0.5", only: :dev, runtime: false} ] end From dfa07dfaae4b62557ce41b61e7414f81c4fe6de7 Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Tue, 3 Sep 2019 00:17:58 +0200 Subject: [PATCH 14/68] Remove ConfigAgent. Gonna keep the options in the state of the genserver. --- lib/exdgraph.ex | 16 ---------------- lib/exdgraph/config_agent.ex | 17 ----------------- 2 files changed, 33 deletions(-) delete mode 100755 lib/exdgraph/config_agent.ex diff --git a/lib/exdgraph.ex b/lib/exdgraph.ex index 619e69d..0ab77b3 100755 --- a/lib/exdgraph.ex +++ b/lib/exdgraph.ex @@ -347,7 +347,6 @@ defmodule ExDgraph do @timeout 15_000 alias ExDgraph.Api.{Mutation, Operation} - alias ExDgraph.{ConfigAgent, Operation, Mutation, Query, Utils} @type conn :: DBConnection.conn() # @type transaction :: DBConnection.t() @@ -428,21 +427,6 @@ defmodule ExDgraph do """ def conn, do: pool_name() - @doc """ - Returns an environment specific ExDgraph configuration. - """ - def config, do: ConfigAgent.get_config() - - @doc false - def config(key), do: Keyword.get(config(), key) - - @doc false - def config(key, default) do - Keyword.get(config(), key, default) - rescue - _ -> default - end - @doc false def pool_name, do: @pool_name diff --git a/lib/exdgraph/config_agent.ex b/lib/exdgraph/config_agent.ex deleted file mode 100755 index 7a7251c..0000000 --- a/lib/exdgraph/config_agent.ex +++ /dev/null @@ -1,17 +0,0 @@ -defmodule ExDgraph.ConfigAgent do - @moduledoc """ - Just hold the user config and offer some utility for accessing it - """ - - use Agent - - @doc false - def start_link(opts) do - Agent.start_link(fn -> %{opts: opts} end, name: __MODULE__) - end - - @doc false - def get_config do - Agent.get(__MODULE__, fn state -> state[:opts] end) - end -end From 0d9a65aa949f53e7ebf681730f1198ced97db7cf Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Tue, 3 Sep 2019 00:18:15 +0200 Subject: [PATCH 15/68] Upgrade Dgraph to 1.0.17. --- docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 6925c19..943bc6a 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: "3.2" services: zero: - image: dgraph/dgraph:v1.0.16 + image: dgraph/dgraph:v1.0.17 volumes: - type: bind source: ./dgraph_data @@ -14,7 +14,7 @@ services: restart: on-failure command: dgraph zero --my=zero:5080 server: - image: dgraph/dgraph:v1.0.16 + image: dgraph/dgraph:v1.0.17 volumes: - type: bind source: ./dgraph_data @@ -27,7 +27,7 @@ services: restart: on-failure command: dgraph alpha --my=server:7080 --lru_mb=2048 --zero=zero:5080 ratel: - image: dgraph/dgraph:v1.0.16 + image: dgraph/dgraph:v1.0.17 volumes: - type: bind source: ./dgraph_data From 7c729753a84d567d1e62e080baa9adf24297d507 Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Tue, 3 Sep 2019 00:18:49 +0200 Subject: [PATCH 16/68] Set show_sensitive_data_on_connection_error to true when testing. --- config/test.exs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/test.exs b/config/test.exs index 9cbb857..221c2d9 100755 --- a/config/test.exs +++ b/config/test.exs @@ -4,4 +4,5 @@ config :ex_dgraph, ExDgraph, # default port considered to be: 9080 hostname: '0.0.0.0', pool_size: 5, - enforce_struct_schema: true + enforce_struct_schema: true, + show_sensitive_data_on_connection_error: true From 0118f4585075a9b9b27f8768330c2f6d761ea990 Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Tue, 3 Sep 2019 00:19:00 +0200 Subject: [PATCH 17/68] Add tls folder to gitignore. --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 5de6e1b..cec5681 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,6 @@ exdgraph-*.tar # Data directory when running Dgraph from docker-compose for test purposes /dgraph_data + +# TLS folder with certificates for testing +/tls/* From aca5eeb02c31a780bb49a95e51ee5baa02c93a86 Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Tue, 3 Sep 2019 00:19:16 +0200 Subject: [PATCH 18/68] Add docker compose file for testing with tls. --- docker-compose-tls.yml | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 docker-compose-tls.yml diff --git a/docker-compose-tls.yml b/docker-compose-tls.yml new file mode 100644 index 0000000..c61e2a0 --- /dev/null +++ b/docker-compose-tls.yml @@ -0,0 +1,42 @@ +version: "3.2" +services: + zero: + image: dgraph/dgraph:v1.0.17 + volumes: + - type: bind + source: ./dgraph_data + target: /dgraph + volume: + nocopy: true + ports: + - 5080:5080 + - 6080:6080 + restart: on-failure + command: dgraph zero --my=zero:5080 + server: + image: dgraph/dgraph:v1.0.17 + volumes: + - type: bind + source: ./dgraph_data + target: /dgraph + volume: + nocopy: true + ports: + - 8080:8080 + - 9080:9080 + restart: on-failure + command: dgraph alpha --my=server:7080 --lru_mb=2048 --zero=zero:5080 --tls_dir tls + ratel: + image: dgraph/dgraph:v1.0.17 + volumes: + - type: bind + source: ./dgraph_data + target: /dgraph + volume: + nocopy: true + ports: + - 8000:8000 + command: dgraph-ratel + +volumes: + dgraph: From 95eb584d6a854056b1e7951a33486ff40aaae80c Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Wed, 4 Sep 2019 23:33:39 +0200 Subject: [PATCH 19/68] Remove QueryStatement. --- lib/exdgraph/query.ex | 30 ++++++++++++++++++++---------- lib/exdgraph/query_statement.ex | 16 ---------------- 2 files changed, 20 insertions(+), 26 deletions(-) delete mode 100644 lib/exdgraph/query_statement.ex diff --git a/lib/exdgraph/query.ex b/lib/exdgraph/query.ex index 545b668..29ad8da 100644 --- a/lib/exdgraph/query.ex +++ b/lib/exdgraph/query.ex @@ -2,7 +2,9 @@ defmodule ExDgraph.Query do @moduledoc """ Provides the functions for the callbacks from the DBConnection behaviour. """ - alias ExDgraph.{Exception, QueryStatement, Transform} + alias ExDgraph.{Exception, Query, Transform} + + defstruct statement: "" @doc false def query(conn, statement) do @@ -10,7 +12,7 @@ defmodule ExDgraph.Query do {:error, f} -> {:error, code: Map.get(f, :code, Map.get(f, :status)), message: f.message} - {:ok, %QueryStatement{}, result} -> + {:ok, %Query{}, result} -> {:ok, result} end end @@ -27,19 +29,27 @@ defmodule ExDgraph.Query do end defp query_commit(conn, statement) do - exec = fn conn -> - q = %QueryStatement{statement: statement} + q = %Query{statement: statement} - case DBConnection.execute(conn, q, %{}) do - {:ok, resp} -> Transform.transform_query(resp) - other -> other - end + case DBConnection.execute(conn, q, %{}) do + {:ok, resp} -> Transform.transform_query(resp) + other -> other end - - DBConnection.run(conn, exec, run_opts()) end defp run_opts do [pool: ExDgraph.config(:pool), timeout: ExDgraph.config(:timeout)] end end + +defimpl DBConnection.Query, for: ExDgraph.Query do + alias ExDgraph.Transform + + def describe(query, _), do: query + + def parse(query, _), do: query + + def encode(_query, data, _), do: data + + def decode(_query, result, _opts), do: Transform.transform_query(result) +end diff --git a/lib/exdgraph/query_statement.ex b/lib/exdgraph/query_statement.ex deleted file mode 100644 index 5af8a32..0000000 --- a/lib/exdgraph/query_statement.ex +++ /dev/null @@ -1,16 +0,0 @@ -defmodule ExDgraph.QueryStatement do - @moduledoc false - defstruct statement: "" -end - -defimpl DBConnection.Query, for: ExDgraph.QueryStatement do - alias ExDgraph.Transform - - def describe(query, _), do: query - - def parse(query, _), do: query - - def encode(_query, data, _), do: data - - def decode(_query, result, _opts), do: Transform.transform_query(result) -end From cefd906996e0971c11c05b09bd3409dd9a44bd98 Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Thu, 5 Sep 2019 23:11:49 +0200 Subject: [PATCH 20/68] Add key :code to Error. --- lib/exdgraph/error.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/exdgraph/error.ex b/lib/exdgraph/error.ex index 2665ac9..40dff47 100755 --- a/lib/exdgraph/error.ex +++ b/lib/exdgraph/error.ex @@ -2,7 +2,7 @@ defmodule ExDgraph.Error do @moduledoc """ Dgraph or connection error are wrapped in ExDgraph.Error. """ - defexception [:reason, :action] + defexception [:reason, :action, :code] @type t :: %ExDgraph.Error{} From 31ee6b729828f9de64a816d4006da6787a8ba063 Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Thu, 5 Sep 2019 23:14:26 +0200 Subject: [PATCH 21/68] Rewrite protocol, keep opts in state and change how ssl configuration is set. --- lib/exdgraph/protocol.ex | 150 ++++++++++++++++++++++++++++++--------- 1 file changed, 117 insertions(+), 33 deletions(-) diff --git a/lib/exdgraph/protocol.ex b/lib/exdgraph/protocol.ex index 4e168c3..14543cd 100755 --- a/lib/exdgraph/protocol.ex +++ b/lib/exdgraph/protocol.ex @@ -16,7 +16,10 @@ defmodule ExDgraph.Protocol do QueryStatement } + alias GRPC.Stub + defstruct [ + :opts, :channel, :connected, :txn_context, @@ -25,19 +28,23 @@ defmodule ExDgraph.Protocol do ] @impl true - def connect(_opts) do - host = to_charlist(ExDgraph.config(:hostname)) - port = ExDgraph.config(:port) + def connect(opts) do + opts = default_opts(opts) + + host = to_charlist(opts[:hostname]) + port = normalize_port(opts[:port]) - opts = - [] - |> set_ssl_opts() - |> Keyword.put(:adapter_opts, %{http2_opts: %{keepalive: ExDgraph.config(:keepalive)}}) + # TODO: with statement? + case gen_stub_options(opts) do + {:ok, stub_opts} -> + case Stub.connect("#{host}:#{port}", stub_opts) do + {:ok, channel} -> + state = %__MODULE__{opts: opts, channel: channel} + {:ok, state} - case GRPC.Stub.connect("#{host}:#{port}", opts) do - {:ok, channel} -> - state = %__MODULE__{channel: channel} - {:ok, state} + {:error, reason} -> + {:error, %Error{action: :connect, reason: reason}} + end {:error, reason} -> {:error, %Error{action: :connect, reason: reason}} @@ -102,17 +109,22 @@ defmodule ExDgraph.Protocol do {:ok, state} end + @impl true + def handle_prepare(query, _opts, %{txn_context: txn_context} = state) do + {:ok, %{query | txn_context: txn_context}, state} + end + @impl true def handle_execute( %QueryStatement{statement: statement} = query, _params, _, %{channel: channel} = state + %{channel: channel, opts: opts} = state ) do request = ExDgraph.Api.Request.new(query: statement) - timeout = ExDgraph.config(:timeout) - case ExDgraph.Api.Dgraph.Stub.query(channel, request, timeout: timeout) do + case ExDgraph.Api.Dgraph.Stub.query(channel, request, timeout: opts[:timeout]) do {:ok, res} -> {:ok, query, res, state} @@ -165,11 +177,11 @@ defmodule ExDgraph.Protocol do _params, _, %{channel: channel} = state + %{channel: channel, opts: opts} = state ) do operation = Api.Operation.new(drop_all: drop_all, schema: schema, drop_attr: drop_attr) - timeout = ExDgraph.config(:timeout) - case ExDgraph.Api.Dgraph.Stub.alter(channel, operation, timeout: timeout) do + case ExDgraph.Api.Dgraph.Stub.alter(channel, operation, timeout: opts[:timeout]) do {:ok, res} -> {:ok, query, res, state} @@ -181,10 +193,8 @@ defmodule ExDgraph.Protocol do {:error, e, state} end - defp do_mutate(%{channel: channel} = state, dgraph_query, query) do - timeout = ExDgraph.config(:timeout) - - case ExDgraph.Api.Dgraph.Stub.mutate(channel, dgraph_query, timeout: timeout) do + defp do_mutate(%{channel: channel, opts: opts} = state, dgraph_query, query) do + case ExDgraph.Api.Dgraph.Stub.mutate(channel, dgraph_query, timeout: opts[:timeout]) do {:ok, res} -> {:ok, query, res, state} @@ -196,31 +206,101 @@ defmodule ExDgraph.Protocol do {:error, e, state} end - defp configure_ssl(ssl_opts \\ []) do - case ExDgraph.config(:ssl) do + @spec default_opts(Keyword.t()) :: Keyword.t() + defp default_opts(opts \\ []) do + opts + |> Keyword.put_new(:hostname, System.get_env("DGRAPH_HOST") || 'localhost') + |> Keyword.put_new(:port, System.get_env("DGRAPH_PORT") || 9080) + |> Keyword.put_new(:name, :ex_dgraph) + |> Keyword.put_new(:timeout, 15_000) + |> Keyword.put_new(:ssl, false) + |> Keyword.put_new(:tls_client_auth, false) + |> Keyword.put_new(:certfile, nil) + |> Keyword.put_new(:keyfile, nil) + |> Keyword.put_new(:cacertfile, nil) + |> Keyword.put_new(:enforce_struct_schema, false) + |> Keyword.put_new(:keepalive, :infinity) + # DBConnection config options + |> Keyword.put_new(:backoff_min, 1_000) + |> Keyword.put_new(:backoff_max, 30_000) + |> Keyword.put_new(:backoff_type, :rand_exp) + |> Keyword.put_new(:pool_size, 5) + |> Keyword.put_new(:idle_interval, 5_000) + |> Keyword.put_new(:max_restarts, 3) + |> Keyword.put_new(:max_seconds, 5) + |> Keyword.update!(:port, &normalize_port/1) + |> Enum.reject(fn {_k, v} -> is_nil(v) end) + end + + def gen_stub_options(opts) do + adapter_opts = %{http2_opts: %{keepalive: opts[:keepalive]}} + stub_opts = [adapter_opts: adapter_opts] + + case gen_ssl_config(opts) do + {:ok, nil} -> + {:ok, stub_opts} + + {:ok, ssl_config} -> + {:ok, Keyword.put(stub_opts, :cred, GRPC.Credential.new(ssl: ssl_config))} + + {:error, error} -> + {:error, error} + end + end + + def gen_ssl_config(opts) do + if opts[:ssl] do + case opts[:cacertfile] do + nil -> + {:error, {:not_provided, :cacertfile}} + + cacertfile -> + with {:ok, tls_config} <- check_tls(opts) do + ssl_config = [{:cacertfile, cacertfile} | tls_config] + ssl_config = for {key, value} <- ssl_config, do: {key, to_charlist(value)} + {:ok, ssl_config} + end + end + else + {:ok, nil} + end + end + + defp check_tls(opts) do + case {opts[:certfile], opts[:keyfile]} do + {nil, nil} -> {:ok, []} + {_, nil} -> {:error, %Error{action: :connect, reason: {:not_provided, :keyfile}}} + {nil, _} -> {:error, %Error{action: :connect, reason: {:not_provided, :certfile}}} + {certfile, keyfile} -> {:ok, [certfile: certfile, keyfile: keyfile]} + end + end + + defp configure_ssl(opts) do + case opts[:ssl] do true -> - add_ssl_file(ssl_opts, :cacertfile) + add_ssl_file(opts, :cacertfile) false -> - ssl_opts + opts end end - defp configure_tls_auth(ssl_opts \\ []) do - case ExDgraph.config(:tls_client_auth) do + defp configure_tls_auth(opts) do + case opts[:tls_client_auth] do true -> - ssl_opts + opts |> add_ssl_file(:certfile) |> add_ssl_file(:keyfile) |> add_ssl_file(:certfile) false -> - ssl_opts + opts end end - defp add_ssl_file(ssl_opts \\ [], type) do - Keyword.put(ssl_opts, type, validate_tls_file(type, ExDgraph.config(type))) + defp add_ssl_file(opts, type) do + path = Keyword.fetch!(opts, type) + Keyword.put(opts, type, validate_tls_file(type, path)) end defp validate_tls_file(type, path) do @@ -231,14 +311,15 @@ defmodule ExDgraph.Protocol do false -> raise Exception, code: 2, - message: "SSL configuration error. File #{type} '#{ExDgraph.config(type)}' not found" + message: "SSL configuration error. File #{type} '#{path}' not found" end end - defp set_ssl_opts(opts \\ []) do - if ExDgraph.config(:ssl) || ExDgraph.config(:tls_client_auth) do + defp set_ssl_opts(opts) do + if opts[:ssl] || opts[:tls_client_auth] do ssl_opts = - configure_ssl() + opts + |> configure_ssl() |> configure_tls_auth() Keyword.put(opts, :cred, GRPC.Credential.new(ssl: ssl_opts)) @@ -246,4 +327,7 @@ defmodule ExDgraph.Protocol do opts end end + + defp normalize_port(port) when is_binary(port), do: String.to_integer(port) + defp normalize_port(port) when is_integer(port), do: port end From 64174887ec9b690315dcdd42b22c112a4aef517e Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Thu, 5 Sep 2019 23:15:33 +0200 Subject: [PATCH 22/68] Move Error to structs file. --- lib/exdgraph/{error.ex => structs.ex} | 1 + 1 file changed, 1 insertion(+) rename lib/exdgraph/{error.ex => structs.ex} (99%) mode change 100755 => 100644 diff --git a/lib/exdgraph/error.ex b/lib/exdgraph/structs.ex old mode 100755 new mode 100644 similarity index 99% rename from lib/exdgraph/error.ex rename to lib/exdgraph/structs.ex index 40dff47..28c453d --- a/lib/exdgraph/error.ex +++ b/lib/exdgraph/structs.ex @@ -11,3 +11,4 @@ defmodule ExDgraph.Error do "#{action} failed with #{inspect(reason)}" end end + From f82d89ed2f5c0e2695929302a442a64a6dfaad37 Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Thu, 5 Sep 2019 23:15:41 +0200 Subject: [PATCH 23/68] Add Result struct. --- lib/exdgraph/structs.ex | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/exdgraph/structs.ex b/lib/exdgraph/structs.ex index 28c453d..fe2540f 100644 --- a/lib/exdgraph/structs.ex +++ b/lib/exdgraph/structs.ex @@ -12,3 +12,10 @@ defmodule ExDgraph.Error do end end +defmodule ExDgraph.Result do + @moduledoc """ + Results from a query are wrapped in ExDgraph.Result + """ + + defstruct [:data, :schema, :txn] +end From 3a5c58f88dfc69decfdf19eccf4d6c2a008b2a97 Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Thu, 5 Sep 2019 23:16:31 +0200 Subject: [PATCH 24/68] Rewrite Query. --- lib/exdgraph/query.ex | 68 +++++++++++++++++---------------------- lib/exdgraph/transform.ex | 20 ------------ 2 files changed, 30 insertions(+), 58 deletions(-) diff --git a/lib/exdgraph/query.ex b/lib/exdgraph/query.ex index 29ad8da..e599a65 100644 --- a/lib/exdgraph/query.ex +++ b/lib/exdgraph/query.ex @@ -2,54 +2,46 @@ defmodule ExDgraph.Query do @moduledoc """ Provides the functions for the callbacks from the DBConnection behaviour. """ - alias ExDgraph.{Exception, Query, Transform} + alias ExDgraph.{Exception, Error, Query, Transform} - defstruct statement: "" - - @doc false - def query(conn, statement) do - case query_commit(conn, statement) do - {:error, f} -> - {:error, code: Map.get(f, :code, Map.get(f, :status)), message: f.message} + defstruct [:statement, :parameters, :txn_context] +end - {:ok, %Query{}, result} -> - {:ok, result} - end - end +defimpl DBConnection.Query, for: ExDgraph.Query do + @moduledoc """ + Implementation of `DBConnection.Query` protocol. + """ - @doc false - def query!(conn, statement) do - case query(conn, statement) do - {:ok, r} -> - r + alias ExDgraph.{Result, Utils} - {:error, code: code, message: message} -> - raise Exception, code: code, message: message - end - end + def describe(query, _), do: query - defp query_commit(conn, statement) do - q = %Query{statement: statement} + @doc """ + Parse a query. - case DBConnection.execute(conn, q, %{}) do - {:ok, resp} -> Transform.transform_query(resp) - other -> other - end - end + This function is called to parse a query term before it is prepared. + """ + def parse(%{statement: nil} = query), do: %{query | statement: ""} - defp run_opts do - [pool: ExDgraph.config(:pool), timeout: ExDgraph.config(:timeout)] + def parse(%{statement: statement} = query, _) do + %{query | statement: IO.iodata_to_binary(statement)} end -end - -defimpl DBConnection.Query, for: ExDgraph.Query do - alias ExDgraph.Transform - def describe(query, _), do: query + def encode(_query, data, _), do: data - def parse(query, _), do: query + def decode(_query, %ExDgraph.Api.Response{json: json, schema: schema, txn: txn} = result, _opts) do + decoded = Jason.decode!(json) - def encode(_query, data, _), do: data + transformed = + case Morphix.atomorphiform(decoded) do + {:ok, parsed} -> parsed + _ -> json + end - def decode(_query, result, _opts), do: Transform.transform_query(result) + %Result{ + data: transformed, + schema: schema, + txn: txn + } + end end diff --git a/lib/exdgraph/transform.ex b/lib/exdgraph/transform.ex index 59b3e4a..350f7f8 100644 --- a/lib/exdgraph/transform.ex +++ b/lib/exdgraph/transform.ex @@ -3,26 +3,6 @@ defmodule ExDgraph.Transform do Transform a raw response from Dgraph. For example decode the json part. """ - @doc """ - Takes a response from Dgraph, parses the `json` with `Poison` and transforms all string - keys into atom keys. - """ - def transform_query(%ExDgraph.Api.Response{json: json, schema: schema, txn: txn}) do - decoded = Jason.decode!(json) - - transformed = - case Morphix.atomorphiform(decoded) do - {:ok, parsed} -> parsed - _ -> json - end - - %{ - result: transformed, - schema: schema, - txn: txn - } - end - @doc """ Takes a response from Dgraph and returns a map. """ From f92a2a707eecc620106730b26e9c6f50a1278c18 Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Thu, 5 Sep 2019 23:16:49 +0200 Subject: [PATCH 25/68] Update test setup. --- test/test_helper.exs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/test_helper.exs b/test/test_helper.exs index 5573444..ef0e2e9 100755 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -60,8 +60,7 @@ defmodule ExDgraph.TestHelper do _:st1 "132" . """ - def import_starwars_sample() do - conn = ExDgraph.conn() + def import_starwars_sample(conn) do ExDgraph.operation(conn, %{schema: @starwars_schema}) {:ok, _} = ExDgraph.mutation(conn, @starwars_creation_mutation) end @@ -77,7 +76,7 @@ defmodule ExDgraph.TestHelper do end if Process.whereis(ExDgraph.pool_name()) == nil do - {:ok, _pid} = ExDgraph.start_link(Application.get_env(:ex_dgraph, ExDgraph)) + {:ok, _conn} = ExDgraph.start_link(Application.get_env(:ex_dgraph, ExDgraph)) end ExUnit.configure(exclude: [tls_tests: true]) From 16bd729da2aeba51a65b916ceb6acd3e6aab408b Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Thu, 5 Sep 2019 23:17:01 +0200 Subject: [PATCH 26/68] Use new test setup in query test. --- test/query_test.exs | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/test/query_test.exs b/test/query_test.exs index a4c2054..3709d2d 100644 --- a/test/query_test.exs +++ b/test/query_test.exs @@ -18,14 +18,9 @@ defmodule ExDgraph.QueryTest do """ setup_all do - conn = ExDgraph.conn() - drop_all() - import_starwars_sample() - - on_exit(fn -> - # close channel ? - :ok - end) + {:ok, conn} = ExDgraph.start_link() + ExDgraph.operation(conn, %{drop_all: true}) + import_starwars_sample(conn) [conn: conn] end From e98fe137a9288c6b9b961c57a17f7b6118c08f88 Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Thu, 5 Sep 2019 23:17:32 +0200 Subject: [PATCH 27/68] Remove default_config from Utils. --- lib/exdgraph/utils.ex | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/lib/exdgraph/utils.ex b/lib/exdgraph/utils.ex index 8f8839a..ec6a26a 100755 --- a/lib/exdgraph/utils.ex +++ b/lib/exdgraph/utils.ex @@ -83,33 +83,6 @@ defmodule ExDgraph.Utils do end end - @doc """ - Fills in the given `opts` with default options. - """ - @spec default_config(Keyword.t()) :: Keyword.t() - def default_config(config \\ Application.get_env(:ex_dgraph, ExDgraph)) do - config - |> Keyword.put_new(:hostname, System.get_env("DGRAPH_HOST") || 'localhost') - |> Keyword.put_new(:port, System.get_env("DGRAPH_PORT") || 9080) - |> Keyword.put_new(:name, :ex_dgraph) - |> Keyword.put_new(:timeout, 15_000) - |> Keyword.put_new(:ssl, false) - |> Keyword.put_new(:tls_client_auth, false) - |> Keyword.put_new(:certfile, nil) - |> Keyword.put_new(:keyfile, nil) - |> Keyword.put_new(:cacertfile, nil) - |> Keyword.put_new(:enforce_struct_schema, false) - |> Keyword.put_new(:keepalive, :infinity) - # DBConnection config options - |> Keyword.put_new(:name, 1_000) - |> Keyword.put_new(:backoff_min, 1_000) - |> Keyword.put_new(:backoff_max, 30_000) - |> Keyword.put_new(:backoff_type, :rand_exp) - |> Keyword.put_new(:pool_size, 5) - |> Keyword.put_new(:idle_interval, 5_000) - |> Keyword.put_new(:max_restarts, 3) - |> Keyword.put_new(:max_seconds, 5) - |> Enum.reject(fn {_k, v} -> is_nil(v) end) end end From 229787c87ccaae14278ea5c9cd0af36d009c9c3f Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Thu, 5 Sep 2019 23:21:28 +0200 Subject: [PATCH 28/68] Remove dependency morphix. --- lib/exdgraph/query.ex | 13 +++++-------- lib/exdgraph/utils.ex | 14 ++++++++++++++ mix.exs | 1 - 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/lib/exdgraph/query.ex b/lib/exdgraph/query.ex index e599a65..57f6830 100644 --- a/lib/exdgraph/query.ex +++ b/lib/exdgraph/query.ex @@ -30,16 +30,13 @@ defimpl DBConnection.Query, for: ExDgraph.Query do def encode(_query, data, _), do: data def decode(_query, %ExDgraph.Api.Response{json: json, schema: schema, txn: txn} = result, _opts) do - decoded = Jason.decode!(json) - - transformed = - case Morphix.atomorphiform(decoded) do - {:ok, parsed} -> parsed - _ -> json - end + data = + json + |> Jason.decode!() + |> Utils.atomify_map_keys() %Result{ - data: transformed, + data: data, schema: schema, txn: txn } diff --git a/lib/exdgraph/utils.ex b/lib/exdgraph/utils.ex index ec6a26a..be8f5e0 100755 --- a/lib/exdgraph/utils.ex +++ b/lib/exdgraph/utils.ex @@ -83,7 +83,21 @@ defmodule ExDgraph.Utils do end end + def atomify_map_keys(map) when is_map(map) do + Enum.reduce(map, %{}, fn + {key, value}, acc when is_atom(key) -> + Map.put(acc, key, atomify_map_keys(value)) + + {key, value}, acc when is_binary(key) -> + Map.put(acc, String.to_existing_atom(key), atomify_map_keys(value)) + end) + end + + def atomify_map_keys(list) when is_list(list) do + for el <- list, do: atomify_map_keys(el) end + + def atomify_map_keys(map), do: map end # Partly Copyright (c) 2017 Jason Goldberger diff --git a/mix.exs b/mix.exs index 9f768c5..2dc18fa 100755 --- a/mix.exs +++ b/mix.exs @@ -41,7 +41,6 @@ defmodule ExDgraph.MixProject do {:elixir_uuid, "~> 1.2"}, {:grpc, "~> 0.3.1"}, {:jason, "~> 1.1"}, - {:morphix, "~> 0.6.0"}, {:protobuf, "~> 0.6.1"}, {:excoveralls, "~> 0.10", only: :test}, {:credo, "~> 1.0", only: [:dev, :test], runtime: false}, From d01398dca9ca2ee4798c564db26b8c29d374104a Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Thu, 5 Sep 2019 23:21:32 +0200 Subject: [PATCH 29/68] Update mix.lock --- mix.lock | 34 ++++++++++++---------------------- 1 file changed, 12 insertions(+), 22 deletions(-) mode change 100755 => 100644 mix.lock diff --git a/mix.lock b/mix.lock old mode 100755 new mode 100644 index e5f1d12..a78c22f --- a/mix.lock +++ b/mix.lock @@ -1,40 +1,30 @@ %{ "bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], [], "hexpm"}, - "certifi": {:hex, :certifi, "2.4.2", "75424ff0f3baaccfd34b1214184b6ef616d89e420b258bb0a5ea7d7bc628f7f0", [:rebar3], [{:parse_trans, "~>3.3", [hex: :parse_trans, repo: "hexpm", optional: false]}], "hexpm"}, - "chatterbox": {:git, "https://github.com/tony612/chatterbox.git", "7fe9b6d89903c59359e4406842cb68ecfa1e9b5c", [branch: "my-fix"]}, + "certifi": {:hex, :certifi, "2.5.1", "867ce347f7c7d78563450a18a6a28a8090331e77fa02380b4a21962a65d36ee5", [:rebar3], [{:parse_trans, "~>3.3", [hex: :parse_trans, repo: "hexpm", optional: false]}], "hexpm"}, "connection": {:hex, :connection, "1.0.4", "a1cae72211f0eef17705aaededacac3eb30e6625b04a6117c1b2db6ace7d5976", [:mix], [], "hexpm"}, "cowboy": {:hex, :cowboy, "2.5.0", "4ef3ae066ee10fe01ea3272edc8f024347a0d3eb95f6fbb9aed556dacbfc1337", [:rebar3], [{:cowlib, "~> 2.6.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "~> 1.6.2", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm"}, "cowlib": {:hex, :cowlib, "2.6.0", "8aa629f81a0fc189f261dc98a42243fa842625feea3c7ec56c48f4ccdb55490f", [:rebar3], [], "hexpm"}, - "credo": {:hex, :credo, "1.0.2", "88bc918f215168bf6ce7070610a6173c45c82f32baa08bdfc80bf58df2d103b6", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, + "credo": {:hex, :credo, "1.1.4", "c2f3b73c895d81d859cec7fcee7ffdb972c595fd8e85ab6f8c2adbf01cf7c29c", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, "db_connection": {:hex, :db_connection, "2.1.1", "a51e8a2ee54ef2ae6ec41a668c85787ed40cb8944928c191280fe34c15b76ae5", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm"}, - "earmark": {:hex, :earmark, "1.3.4", "52aba89c60529284df5fc18adc4c808b7346e72668bc2fb2b68d7394996c4af8", [:mix], [], "hexpm"}, + "earmark": {:hex, :earmark, "1.4.0", "397e750b879df18198afc66505ca87ecf6a96645545585899f6185178433cc09", [:mix], [], "hexpm"}, "elixir_uuid": {:hex, :elixir_uuid, "1.2.0", "ff26e938f95830b1db152cb6e594d711c10c02c6391236900ddd070a6b01271d", [:mix], [], "hexpm"}, - "ex_doc": {:hex, :ex_doc, "0.19.3", "3c7b0f02851f5fc13b040e8e925051452e41248f685e40250d7e40b07b9f8c10", [:mix], [{:earmark, "~> 1.2", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.10", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm"}, - "excoveralls": {:hex, :excoveralls, "0.10.5", "7c912c4ec0715a6013647d835c87cde8154855b9b84e256bc7a63858d5f284e3", [:mix], [{:hackney, "~> 1.13", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, - "exjsx": {:hex, :exjsx, "4.0.0", "60548841e0212df401e38e63c0078ec57b33e7ea49b032c796ccad8cde794b5c", [:mix], [{:jsx, "~> 2.8.0", [hex: :jsx, repo: "hexpm", optional: false]}], "hexpm"}, - "file_system": {:hex, :file_system, "0.2.6", "fd4dc3af89b9ab1dc8ccbcc214a0e60c41f34be251d9307920748a14bf41f1d3", [:mix], [], "hexpm"}, - "fs": {:hex, :fs, "0.9.2", "ed17036c26c3f70ac49781ed9220a50c36775c6ca2cf8182d123b6566e49ec59", [:rebar], [], "hexpm"}, + "ex_doc": {:hex, :ex_doc, "0.21.2", "caca5bc28ed7b3bdc0b662f8afe2bee1eedb5c3cf7b322feeeb7c6ebbde089d6", [:mix], [{:earmark, "~> 1.3.3 or ~> 1.4", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm"}, + "excoveralls": {:hex, :excoveralls, "0.11.2", "0c6f2c8db7683b0caa9d490fb8125709c54580b4255ffa7ad35f3264b075a643", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, + "file_system": {:hex, :file_system, "0.2.7", "e6f7f155970975789f26e77b8b8d8ab084c59844d8ecfaf58cbda31c494d14aa", [:mix], [], "hexpm"}, "grpc": {:hex, :grpc, "0.3.1", "bba240631f1a262db865d9bc620b3e3abc0acfab27a922ad47727d057a734ab3", [:mix], [{:cowboy, "~> 2.5", [hex: :cowboy, repo: "hexpm", optional: false]}, {:gun, "~> 1.2", [hex: :gun, repo: "hexpm", optional: false]}, {:protobuf, "~> 0.5", [hex: :protobuf, repo: "hexpm", optional: false]}], "hexpm"}, "gun": {:hex, :gun, "1.3.0", "18e5d269649c987af95aec309f68a27ffc3930531dd227a6eaa0884d6684286e", [:rebar3], [{:cowlib, "~> 2.6.0", [hex: :cowlib, repo: "hexpm", optional: false]}], "hexpm"}, - "hackney": {:hex, :hackney, "1.15.0", "287a5d2304d516f63e56c469511c42b016423bcb167e61b611f6bad47e3ca60e", [:rebar3], [{:certifi, "2.4.2", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "1.0.2", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.4", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"}, - "hpack": {:git, "https://github.com/joedevivo/hpack.git", "6b58b6231e9b6ab83096715120578976f72f4f7c", [tag: "0.2.3"]}, + "hackney": {:hex, :hackney, "1.15.1", "9f8f471c844b8ce395f7b6d8398139e26ddca9ebc171a8b91342ee15a19963f4", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.4", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"}, "idna": {:hex, :idna, "6.0.0", "689c46cbcdf3524c44d5f3dde8001f364cd7608a99556d8fbd8239a5798d4c10", [:rebar3], [{:unicode_util_compat, "0.4.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm"}, "jason": {:hex, :jason, "1.1.2", "b03dedea67a99223a2eaf9f1264ce37154564de899fd3d8b9a21b1a6fd64afe7", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm"}, - "jsx": {:hex, :jsx, "2.8.3", "a05252d381885240744d955fbe3cf810504eb2567164824e19303ea59eef62cf", [:mix, :rebar3], [], "hexpm"}, - "makeup": {:hex, :makeup, "0.5.5", "9e08dfc45280c5684d771ad58159f718a7b5788596099bdfb0284597d368a882", [:mix], [{:nimble_parsec, "~> 0.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm"}, - "makeup_elixir": {:hex, :makeup_elixir, "0.10.0", "0f09c2ddf352887a956d84f8f7e702111122ca32fbbc84c2f0569b8b65cbf7fa", [:mix], [{:makeup, "~> 0.5.5", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm"}, + "makeup": {:hex, :makeup, "1.0.0", "671df94cf5a594b739ce03b0d0316aa64312cee2574b6a44becb83cd90fb05dc", [:mix], [{:nimble_parsec, "~> 0.5.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm"}, + "makeup_elixir": {:hex, :makeup_elixir, "0.14.0", "cf8b7c66ad1cff4c14679698d532f0b5d45a3968ffbcbfd590339cb57742f1ae", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm"}, "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm"}, - "mimerl": {:hex, :mimerl, "1.0.2", "993f9b0e084083405ed8252b99460c4f0563e41729ab42d9074fd5e52439be88", [:rebar3], [], "hexpm"}, + "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm"}, "mix_test_watch": {:hex, :mix_test_watch, "0.9.0", "c72132a6071261893518fa08e121e911c9358713f62794a90c95db59042af375", [:mix], [{:file_system, "~> 0.2.1 or ~> 0.3", [hex: :file_system, repo: "hexpm", optional: false]}], "hexpm"}, - "morphix": {:hex, :morphix, "0.6.0", "78902c71672a5de64759fb5847f9bed708eb82bc0eb017f1e5cd95f93f1a167b", [:mix], [], "hexpm"}, - "nimble_parsec": {:hex, :nimble_parsec, "0.4.0", "ee261bb53214943679422be70f1658fff573c5d0b0a1ecd0f18738944f818efe", [:mix], [], "hexpm"}, + "nimble_parsec": {:hex, :nimble_parsec, "0.5.1", "c90796ecee0289dbb5ad16d3ad06f957b0cd1199769641c961cfe0b97db190e0", [:mix], [], "hexpm"}, "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm"}, - "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm"}, - "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm"}, - "protobuf": {:hex, :protobuf, "0.6.1", "3ae3ca63d12d1a8c5b10c2e5fbc5145dccbff842ccd541109b74b20b631cea3b", [:mix], [], "hexpm"}, + "protobuf": {:hex, :protobuf, "0.6.3", "742e9034c20532534ca96d7a7ee1251e0f2327dcf8ea6de0dd3eb3e054b236a7", [:mix], [], "hexpm"}, "ranch": {:hex, :ranch, "1.6.2", "6db93c78f411ee033dbb18ba8234c5574883acb9a75af0fb90a9b82ea46afa00", [:rebar3], [], "hexpm"}, - "retry": {:hex, :retry, "0.13.0", "bb9b2713f70f39337837852337ad280c77662574f4fb852a8386c269f3d734c4", [:mix], [], "hexpm"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"}, "unicode_util_compat": {:hex, :unicode_util_compat, "0.4.1", "d869e4c68901dd9531385bb0c8c40444ebf624e60b6962d95952775cac5e90cd", [:rebar3], [], "hexpm"}, - "uuid": {:hex, :uuid, "1.1.8", "e22fc04499de0de3ed1116b770c7737779f226ceefa0badb3592e64d5cfb4eb9", [:mix], [], "hexpm"}, } From c943253a17732bdea53671cfde50322184455b66 Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Thu, 5 Sep 2019 23:25:01 +0200 Subject: [PATCH 30/68] Test connection errors. --- test/exdgraph_test.exs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/test/exdgraph_test.exs b/test/exdgraph_test.exs index fb49e26..a2fec2e 100755 --- a/test/exdgraph_test.exs +++ b/test/exdgraph_test.exs @@ -1,3 +1,22 @@ defmodule ExDgraphTest do use ExUnit.Case + import ExUnit.CaptureLog + + test "connection_errors" do + Process.flag(:trap_exit, true) + opts = [backoff_type: :stop, max_restarts: 0, timeout: 500] + + assert capture_log(fn -> + {:ok, pid} = ExDgraph.start_link([hostname: "non_existing"] ++ opts) + assert_receive {:EXIT, ^pid, :killed}, 10000 + end) =~ + "** (ExDgraph.Error) connect failed with \"Error when opening connection: :timeout\"" + + assert capture_log(fn -> + {:ok, pid} = ExDgraph.start_link([port: 700] ++ opts) + + assert_receive {:EXIT, ^pid, :killed}, 10000 + end) =~ + "** (ExDgraph.Error) connect failed with \"Error when opening connection: :timeout\"" + end end From 104a6b12f9aa54dd2709292e58c05ab01f7e44c3 Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Thu, 5 Sep 2019 23:26:10 +0200 Subject: [PATCH 31/68] Add ProtocolTest and test ssl errors. --- test/protocol_test.exs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 test/protocol_test.exs diff --git a/test/protocol_test.exs b/test/protocol_test.exs new file mode 100644 index 0000000..1aa1a18 --- /dev/null +++ b/test/protocol_test.exs @@ -0,0 +1,19 @@ +defmodule ExDgraph.ProtocolTest do + use ExUnit.Case, async: true + alias ExDgraph.Error + + test "ssl_connection_errors" do + opts = [ + backoff_type: :stop, + ssl: true + ] + + Process.flag(:trap_exit, true) + + assert {:error, + %Error{ + action: :connect, + reason: {:not_provided, :cacertfile} + }} = ExDgraph.Protocol.connect(opts) + end +end From 2ab83f8463039877e18cfe06ba3e16c17b8e9b2c Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Thu, 5 Sep 2019 23:26:53 +0200 Subject: [PATCH 32/68] Rewrite start_link in ExDgraph. --- lib/exdgraph.ex | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/lib/exdgraph.ex b/lib/exdgraph.ex index 0ab77b3..7dbc5ac 100755 --- a/lib/exdgraph.ex +++ b/lib/exdgraph.ex @@ -343,14 +343,20 @@ defmodule ExDgraph do use Supervisor - @pool_name :ex_dgraph - @timeout 15_000 - alias ExDgraph.Api.{Mutation, Operation} + alias ExDgraph.{Operation, Mutation, Protocol, Query, Utils} @type conn :: DBConnection.conn() # @type transaction :: DBConnection.t() + @pool_name :ex_dgraph + + # Inherited from DBConnection + + @idle_timeout 5000 + @pool_timeout 5000 + @timeout 15000 + @doc """ Start the connection process and connect to Dgraph @@ -361,6 +367,8 @@ defmodule ExDgraph do - `:password` - User password; - `:pool_size` - maximum pool size; - `:timeout` - Connect timeout in milliseconds (default: `#{@timeout}`) for DBConnection and the GRPC client deadline. + - `:idle_timeout` - Idle timeout to ping database to maintain a connection + (default: `#{@idle_timeout}`) - `:ssl` - If to use ssl for the connection (please see configuration example). If you set this option, you also have to set `cacertfile` to the correct path. - `:tls_client_auth` - If to use TLS client authentication for the connection @@ -406,20 +414,8 @@ defmodule ExDgraph do {:ok, pid} = ExDgraph.start_link(opts) """ @spec start_link(Keyword.t()) :: Supervisor.on_start() - def start_link(opts) do - Supervisor.start_link(__MODULE__, opts, name: __MODULE__) - end - - @doc false - def init(opts) do - cnf = Utils.default_config(opts) - - children = [ - {ExDgraph.ConfigAgent, cnf}, - DBConnection.child_spec(ExDgraph.Protocol, cnf) - ] - - Supervisor.init(children, strategy: :one_for_one) + def start_link(opts \\ []) do + DBConnection.start_link(Protocol, opts) end @doc """ From a8eadfd4f60b516d7c9eff524b5c76220e195fa5 Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Thu, 5 Sep 2019 23:27:10 +0200 Subject: [PATCH 33/68] Move query to ExDgraph. --- lib/exdgraph.ex | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/lib/exdgraph.ex b/lib/exdgraph.ex index 7dbc5ac..7d1acd2 100755 --- a/lib/exdgraph.ex +++ b/lib/exdgraph.ex @@ -489,15 +489,29 @@ defmodule ExDgraph do {:error, [code: 2, message: "while lexing invalid_statement: Invalid operation type: invalid_statement"]} """ - @spec query(conn, String.t()) :: {:ok, ExDgraph.Response} | {:error, ExDgraph.Error} - defdelegate query(conn, statement), to: Query + @spec query(conn, iodata, map, Keyword.t()) :: {:ok, map} | {:error, Dgraph.Error.t() | term} + def query(conn, statement, parameters \\ %{}, opts \\ []) do + query = %Query{statement: statement} + + with {:ok, %Query{} = query, result} <- + DBConnection.prepare_execute(conn, query, parameters, opts), + do: {:ok, query, result} + end @doc """ The same as `query/2` but raises a ExDgraph.Exception if it fails. Returns the server response otherwise. """ @spec query!(conn, String.t()) :: ExDgraph.Response | ExDgraph.Exception - defdelegate query!(conn, statement), to: Query + def query!(conn, statement) do + case query(conn, statement) do + {:ok, r} -> + r + + {:error, code: code, message: message} -> + raise Exception, code: code, message: message + end + end ## Mutation ###################### From 1079d47e3c639c250b0a8491a7b6663aab0fe51b Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Thu, 5 Sep 2019 23:44:48 +0200 Subject: [PATCH 34/68] Use to_atom instead of to_existing_atom. --- lib/exdgraph/utils.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/exdgraph/utils.ex b/lib/exdgraph/utils.ex index be8f5e0..2385e6b 100755 --- a/lib/exdgraph/utils.ex +++ b/lib/exdgraph/utils.ex @@ -89,7 +89,7 @@ defmodule ExDgraph.Utils do Map.put(acc, key, atomify_map_keys(value)) {key, value}, acc when is_binary(key) -> - Map.put(acc, String.to_existing_atom(key), atomify_map_keys(value)) + Map.put(acc, String.to_atom(key), atomify_map_keys(value)) end) end From 0fb4a649e53f772942cf3399d1407c7407d4d9ec Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Fri, 6 Sep 2019 00:04:46 +0200 Subject: [PATCH 35/68] Remove QueryStatement from Protocol. --- lib/exdgraph/protocol.ex | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/exdgraph/protocol.ex b/lib/exdgraph/protocol.ex index 14543cd..d1a0bc7 100755 --- a/lib/exdgraph/protocol.ex +++ b/lib/exdgraph/protocol.ex @@ -13,7 +13,7 @@ defmodule ExDgraph.Protocol do Exception, MutationStatement, OperationStatement, - QueryStatement + Query } alias GRPC.Stub @@ -116,10 +116,9 @@ defmodule ExDgraph.Protocol do @impl true def handle_execute( - %QueryStatement{statement: statement} = query, - _params, - _, - %{channel: channel} = state + %Query{statement: statement} = query, + _request, + _opts, %{channel: channel, opts: opts} = state ) do request = ExDgraph.Api.Request.new(query: statement) From 5c48429adc29f17a5488e66f2523cc1480dbd5fe Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Fri, 6 Sep 2019 00:05:22 +0200 Subject: [PATCH 36/68] Return ExDgraph.Error and state from handle_prepare. --- lib/exdgraph/protocol.ex | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/exdgraph/protocol.ex b/lib/exdgraph/protocol.ex index d1a0bc7..1a88345 100755 --- a/lib/exdgraph/protocol.ex +++ b/lib/exdgraph/protocol.ex @@ -127,12 +127,9 @@ defmodule ExDgraph.Protocol do {:ok, res} -> {:ok, query, res, state} - {:error, f} -> - raise Exception, code: f.status, message: f.message + {:error, error} -> + {:error, %Error{action: :query, code: error.status, reason: error.message}, state} end - rescue - e -> - {:error, e, state} end @impl true From d684da8c6145ba7cd930c3c8fb7f0597c23ad95c Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Fri, 6 Sep 2019 00:39:13 +0200 Subject: [PATCH 37/68] Properly raise for query!/2. --- lib/exdgraph.ex | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/exdgraph.ex b/lib/exdgraph.ex index 7d1acd2..3030bed 100755 --- a/lib/exdgraph.ex +++ b/lib/exdgraph.ex @@ -505,11 +505,11 @@ defmodule ExDgraph do @spec query!(conn, String.t()) :: ExDgraph.Response | ExDgraph.Exception def query!(conn, statement) do case query(conn, statement) do - {:ok, r} -> - r + {:ok, query, result} -> + result - {:error, code: code, message: message} -> - raise Exception, code: code, message: message + {:error, error} -> + raise error end end From 649cc3df5d9f54b9735fb12e0484ae137b113fc8 Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Fri, 6 Sep 2019 00:39:29 +0200 Subject: [PATCH 38/68] Fix query tests. --- test/exdgraph_test.exs | 91 ++++++++++++++++++++++++++++++++++++------ test/query_test.exs | 58 --------------------------- 2 files changed, 78 insertions(+), 71 deletions(-) delete mode 100644 test/query_test.exs diff --git a/test/exdgraph_test.exs b/test/exdgraph_test.exs index a2fec2e..9390338 100755 --- a/test/exdgraph_test.exs +++ b/test/exdgraph_test.exs @@ -1,22 +1,87 @@ defmodule ExDgraphTest do use ExUnit.Case + import ExDgraph.TestHelper import ExUnit.CaptureLog - test "connection_errors" do - Process.flag(:trap_exit, true) - opts = [backoff_type: :stop, max_restarts: 0, timeout: 500] + alias ExDgraph.{Error, Query, Result} - assert capture_log(fn -> - {:ok, pid} = ExDgraph.start_link([hostname: "non_existing"] ++ opts) - assert_receive {:EXIT, ^pid, :killed}, 10000 - end) =~ - "** (ExDgraph.Error) connect failed with \"Error when opening connection: :timeout\"" + setup_all do + {:ok, conn} = ExDgraph.start_link() + ExDgraph.operation(conn, %{drop_all: true}) + import_starwars_sample(conn) - assert capture_log(fn -> - {:ok, pid} = ExDgraph.start_link([port: 700] ++ opts) + [conn: conn] + end + + describe "connection" do + test "connection_errors" do + Process.flag(:trap_exit, true) + opts = [backoff_type: :stop, max_restarts: 0, timeout: 500] + + assert capture_log(fn -> + {:ok, pid} = ExDgraph.start_link([hostname: "non_existing"] ++ opts) + assert_receive {:EXIT, ^pid, :killed}, 10000 + end) =~ + "** (ExDgraph.Error) connect failed with \"Error when opening connection: :timeout\"" + + assert capture_log(fn -> + {:ok, pid} = ExDgraph.start_link([port: 700] ++ opts) + + assert_receive {:EXIT, ^pid, :killed}, 10000 + end) =~ + "** (ExDgraph.Error) connect failed with \"Error when opening connection: :timeout\"" + end + end + + describe "Query" do + @sample_query """ + { + starwars(func: anyofterms(name, "VI")) + { + uid + name + release_date + starring + { + name + } + } + } + """ + + test "query/2 with correct query returns {:ok, %Query{}, %Result{}}", %{conn: conn} do + {status, %Query{} = _query, %Result{} = result} = ExDgraph.query(conn, @sample_query) + assert status == :ok + data = result.data + starwars = List.first(data.starwars) + assert starwars.name == "Star Wars: Episode VI - Return of the Jedi" + assert starwars.release_date == "1983-05-25" + assert length(starwars.starring) == 3 + end + + test "query/2 with wrong query returns {:error, %Error{}}", %{conn: conn} do + {status, %Error{} = error} = ExDgraph.query(conn, "wrong") + assert status == :error + + assert %Error{} = error + assert error.code == 2 + assert error.action == :query + assert error.reason =~ "Invalid operation type: wrong" + end + + test "query!/2 with correct query returns query_msg", %{conn: conn} do + %Result{} = result = ExDgraph.query!(conn, @sample_query) + data = result.data + starwars = List.first(data.starwars) + assert starwars.name == "Star Wars: Episode VI - Return of the Jedi" + assert starwars.release_date == "1983-05-25" + assert length(starwars.starring) == 3 + end - assert_receive {:EXIT, ^pid, :killed}, 10000 - end) =~ - "** (ExDgraph.Error) connect failed with \"Error when opening connection: :timeout\"" + test "query!/2 raises ExDgraph.Exception", %{conn: conn} do + assert_raise ExDgraph.Error, fn -> + ExDgraph.query!(conn, "wrong") + end + end end end diff --git a/test/query_test.exs b/test/query_test.exs deleted file mode 100644 index 3709d2d..0000000 --- a/test/query_test.exs +++ /dev/null @@ -1,58 +0,0 @@ -defmodule ExDgraph.QueryTest do - use ExUnit.Case, async: true - import ExDgraph.TestHelper - - @sample_query """ - { - starwars(func: anyofterms(name, "VI")) - { - uid - name - release_date - starring - { - name - } - } - } - """ - - setup_all do - {:ok, conn} = ExDgraph.start_link() - ExDgraph.operation(conn, %{drop_all: true}) - import_starwars_sample(conn) - - [conn: conn] - end - - test "query/2 with correct query returns {:ok, query_msg}", %{conn: conn} do - {status, query_msg} = ExDgraph.Query.query(conn, @sample_query) - assert status == :ok - res = query_msg.result - starwars = res[:starwars] - one = List.first(starwars) - assert "Star Wars: Episode VI - Return of the Jedi" == one[:name] - assert "1983-05-25" == one[:release_date] - end - - test "query/2 with wrong query returns {:error, error}", %{conn: conn} do - {status, error} = ExDgraph.Query.query(conn, "wrong") - assert status == :error - assert error == [code: 2, message: "while lexing wrong: Invalid operation type: wrong"] - end - - test "query!/2 with correct query returns query_msg", %{conn: conn} do - query_msg = ExDgraph.Query.query!(conn, @sample_query) - res = query_msg.result - starwars = res[:starwars] - one = List.first(starwars) - assert "Star Wars: Episode VI - Return of the Jedi" == one[:name] - assert "1983-05-25" == one[:release_date] - end - - test "query!/2 raises ExDgraph.Exception", %{conn: conn} do - assert_raise ExDgraph.Exception, fn -> - ExDgraph.Query.query!(conn, "wrong") - end - end -end From fab9427ba6a7a019c7150f6a04d61c47ffc75047 Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Fri, 6 Sep 2019 00:44:45 +0200 Subject: [PATCH 39/68] Test start/2 --- test/application_test.exs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/test/application_test.exs b/test/application_test.exs index 7a0c9c8..161e6e8 100644 --- a/test/application_test.exs +++ b/test/application_test.exs @@ -1,13 +1,10 @@ defmodule ApplicationTest do use ExUnit.Case - test "Make sure the application is running" do - res = Application.ensure_started(:ex_dgraph) - assert res == :ok - cnf = ExDgraph.Utils.default_config() - {state, {message, _}} = ExDgraph.Application.start(%{}, cnf) - assert state == :error - assert message == :already_started + test "start/2 starts the application" do + {status, pid} = ExDgraph.Application.start(nil, []) + assert status == :ok + assert Process.alive?(pid) end test "stop/1 returns :ok" do From 846ba4933f56e7963ab7d6be644a7a204c31445f Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Fri, 6 Sep 2019 00:58:05 +0200 Subject: [PATCH 40/68] Add tests for ExDgraph.start_link/1. --- test/exdgraph_test.exs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/test/exdgraph_test.exs b/test/exdgraph_test.exs index 9390338..29e4e3e 100755 --- a/test/exdgraph_test.exs +++ b/test/exdgraph_test.exs @@ -13,6 +13,23 @@ defmodule ExDgraphTest do [conn: conn] end + describe "start_link/1" do + test "it starts the client" do + {status, pid} = ExDgraph.start_link() + assert status == :ok + assert Process.alive?(pid) + end + + test "it starts the process in the DBConnection.ConnectionPool" do + {status, pid} = ExDgraph.start_link() + assert status == :ok + assert Process.alive?(pid) + process_as_map = Process.info(pid) |> Enum.into(%{}) + {ancestor, :init, _} = process_as_map.dictionary[:"$initial_call"] + assert ancestor == DBConnection.ConnectionPool + end + end + describe "connection" do test "connection_errors" do Process.flag(:trap_exit, true) From 31c6a4d6d501b785b6df55c6d0822625112b5355 Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Fri, 6 Sep 2019 00:58:44 +0200 Subject: [PATCH 41/68] Cleanup. --- lib/exdgraph/protocol.ex | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/exdgraph/protocol.ex b/lib/exdgraph/protocol.ex index 1a88345..d918645 100755 --- a/lib/exdgraph/protocol.ex +++ b/lib/exdgraph/protocol.ex @@ -146,8 +146,8 @@ defmodule ExDgraph.Protocol do @impl true def handle_execute( %MutationStatement{statement: "", set_json: set_json} = query, - _params, - _, + _request, + _opts, state ) do dgraph_query = ExDgraph.Api.Mutation.new(set_json: set_json, commit_now: true) @@ -172,7 +172,6 @@ defmodule ExDgraph.Protocol do %OperationStatement{drop_all: drop_all, schema: schema, drop_attr: drop_attr} = query, _params, _, - %{channel: channel} = state %{channel: channel, opts: opts} = state ) do operation = Api.Operation.new(drop_all: drop_all, schema: schema, drop_attr: drop_attr) From 3032561090127342a73a76abb5b6ad99d7191365 Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Fri, 6 Sep 2019 01:14:00 +0200 Subject: [PATCH 42/68] Add typespecs. --- lib/exdgraph/structs.ex | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/exdgraph/structs.ex b/lib/exdgraph/structs.ex index fe2540f..f53e0ef 100644 --- a/lib/exdgraph/structs.ex +++ b/lib/exdgraph/structs.ex @@ -4,7 +4,11 @@ defmodule ExDgraph.Error do """ defexception [:reason, :action, :code] - @type t :: %ExDgraph.Error{} + @type t :: %ExDgraph.Error{ + reason: String.t(), + action: atom(), + code: non_neg_integer + } @impl true def message(%{action: action, reason: reason}) do @@ -17,5 +21,11 @@ defmodule ExDgraph.Result do Results from a query are wrapped in ExDgraph.Result """ + @type t :: %__MODULE__{ + data: %{optional(any) => any}, + schema: [any()], + txn: %ExDgraph.Api.TxnContext{} + } + defstruct [:data, :schema, :txn] end From 88354e34fc208c72140b799f03b9962a5590f034 Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Fri, 6 Sep 2019 01:15:19 +0200 Subject: [PATCH 43/68] Fix warnings. --- lib/exdgraph/query.ex | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/exdgraph/query.ex b/lib/exdgraph/query.ex index 57f6830..8d9d0e7 100644 --- a/lib/exdgraph/query.ex +++ b/lib/exdgraph/query.ex @@ -2,7 +2,6 @@ defmodule ExDgraph.Query do @moduledoc """ Provides the functions for the callbacks from the DBConnection behaviour. """ - alias ExDgraph.{Exception, Error, Query, Transform} defstruct [:statement, :parameters, :txn_context] end @@ -29,7 +28,11 @@ defimpl DBConnection.Query, for: ExDgraph.Query do def encode(_query, data, _), do: data - def decode(_query, %ExDgraph.Api.Response{json: json, schema: schema, txn: txn} = result, _opts) do + def decode( + _query, + %ExDgraph.Api.Response{json: json, schema: schema, txn: txn} = _result, + _opts + ) do data = json |> Jason.decode!() From 976057d8c269ca5959f958899ad44f1ba40c5b63 Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Fri, 6 Sep 2019 13:26:19 +0200 Subject: [PATCH 44/68] Update docs and specs. --- lib/exdgraph.ex | 18 +++++++++--------- test/exdgraph_test.exs | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/exdgraph.ex b/lib/exdgraph.ex index 3030bed..0449b95 100755 --- a/lib/exdgraph.ex +++ b/lib/exdgraph.ex @@ -489,7 +489,7 @@ defmodule ExDgraph do {:error, [code: 2, message: "while lexing invalid_statement: Invalid operation type: invalid_statement"]} """ - @spec query(conn, iodata, map, Keyword.t()) :: {:ok, map} | {:error, Dgraph.Error.t() | term} + @spec query(conn, iodata, map, Keyword.t()) :: {:ok, map} | {:error, ExDgraph.Error.t() | term} def query(conn, statement, parameters \\ %{}, opts \\ []) do query = %Query{statement: statement} @@ -499,10 +499,10 @@ defmodule ExDgraph do end @doc """ - The same as `query/2` but raises a ExDgraph.Exception if it fails. + The same as `query/3` but raises a ExDgraph.Exception if it fails. Returns the server response otherwise. """ - @spec query!(conn, String.t()) :: ExDgraph.Response | ExDgraph.Exception + @spec query!(conn, String.t()) :: ExDgraph.Result | ExDgraph.Error def query!(conn, statement) do case query(conn, statement) do {:ok, query, result} -> @@ -591,14 +591,14 @@ defmodule ExDgraph do } """ - @spec mutation(conn, String.t()) :: {:ok, ExDgraph.Response} | {:error, ExDgraph.Error} + @spec mutation(conn, String.t()) :: {:ok, ExDgraph.Result} | {:error, ExDgraph.Error} defdelegate mutation(conn, statement), to: Mutation @doc """ The same as `mutation/2` but raises an `ExDgraph.Exception` if it fails. Returns the server response otherwise. """ - @spec mutation!(conn, String.t()) :: ExDgraph.Response | ExDgraph.Exception + @spec mutation!(conn, String.t()) :: ExDgraph.Result | ExDgraph.Exception defdelegate mutation!(conn, statement), to: Mutation @doc """ @@ -638,14 +638,14 @@ defmodule ExDgraph do } } """ - @spec set_map(conn, Map.t()) :: {:ok, ExDgraph.Response} | {:error, ExDgraph.Error} + @spec set_map(conn, Map.t()) :: {:ok, ExDgraph.Result} | {:error, ExDgraph.Error} defdelegate set_map(conn, map), to: Mutation @doc """ The same as `set_map/2` but raises an `ExDgraph.Exception` if it fails. Returns the server response otherwise. """ - @spec set_map!(conn, Map.t()) :: {:ok, ExDgraph.Response} | {:error, ExDgraph.Error} + @spec set_map!(conn, Map.t()) :: {:ok, ExDgraph.Result} | {:error, ExDgraph.Error} defdelegate set_map!(conn, map), to: Mutation @doc """ @@ -691,14 +691,14 @@ defmodule ExDgraph do } } """ - @spec set_struct(conn, Map.t()) :: {:ok, ExDgraph.Response} | {:error, ExDgraph.Error} + @spec set_struct(conn, Map.t()) :: {:ok, ExDgraph.Result} | {:error, ExDgraph.Error} defdelegate set_struct(conn, map), to: Mutation @doc """ The same as `set_struct/2` but raises an `ExDgraph.Exception` if it fails. Returns the server response otherwise. """ - @spec set_struct!(conn, Map.t()) :: {:ok, ExDgraph.Response} | {:error, ExDgraph.Error} + @spec set_struct!(conn, Map.t()) :: {:ok, ExDgraph.Result} | {:error, ExDgraph.Error} defdelegate set_struct!(conn, map), to: Mutation ## Operation diff --git a/test/exdgraph_test.exs b/test/exdgraph_test.exs index 29e4e3e..3ed4701 100755 --- a/test/exdgraph_test.exs +++ b/test/exdgraph_test.exs @@ -66,7 +66,7 @@ defmodule ExDgraphTest do } """ - test "query/2 with correct query returns {:ok, %Query{}, %Result{}}", %{conn: conn} do + test "query/3 with correct query returns {:ok, %Query{}, %Result{}}", %{conn: conn} do {status, %Query{} = _query, %Result{} = result} = ExDgraph.query(conn, @sample_query) assert status == :ok data = result.data @@ -76,7 +76,7 @@ defmodule ExDgraphTest do assert length(starwars.starring) == 3 end - test "query/2 with wrong query returns {:error, %Error{}}", %{conn: conn} do + test "query/3 with wrong query returns {:error, %Error{}}", %{conn: conn} do {status, %Error{} = error} = ExDgraph.query(conn, "wrong") assert status == :error From d635df58818774f35988e2b7ffe4e2a19555c871 Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Fri, 6 Sep 2019 13:26:28 +0200 Subject: [PATCH 45/68] Add Payload struct. --- lib/exdgraph/structs.ex | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lib/exdgraph/structs.ex b/lib/exdgraph/structs.ex index f53e0ef..e034cce 100644 --- a/lib/exdgraph/structs.ex +++ b/lib/exdgraph/structs.ex @@ -29,3 +29,15 @@ defmodule ExDgraph.Result do defstruct [:data, :schema, :txn] end + +defmodule ExDgraph.Payload do + @moduledoc """ + Results from alter are wrapped in ExDgraph.Payload + """ + + @type t :: %__MODULE__{ + data: %{optional(any) => any} + } + + defstruct [:data] +end From 30b8dfe3047c1a747164a3d554a2c2990e174176 Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Fri, 6 Sep 2019 13:27:06 +0200 Subject: [PATCH 46/68] Use correct types and explicit parameter names in callbacks. --- lib/exdgraph/query.ex | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/exdgraph/query.ex b/lib/exdgraph/query.ex index 8d9d0e7..83a1e48 100644 --- a/lib/exdgraph/query.ex +++ b/lib/exdgraph/query.ex @@ -11,22 +11,22 @@ defimpl DBConnection.Query, for: ExDgraph.Query do Implementation of `DBConnection.Query` protocol. """ - alias ExDgraph.{Result, Utils} + alias ExDgraph.{Query, Result, Utils} - def describe(query, _), do: query + def describe(query, _opts), do: query @doc """ Parse a query. This function is called to parse a query term before it is prepared. """ - def parse(%{statement: nil} = query), do: %{query | statement: ""} + def parse(%{statement: nil} = query, _opts), do: %Query{query | statement: ""} - def parse(%{statement: statement} = query, _) do - %{query | statement: IO.iodata_to_binary(statement)} + def parse(%{statement: statement} = query, _opts) do + %Query{query | statement: IO.iodata_to_binary(statement)} end - def encode(_query, data, _), do: data + def encode(_query, data, _opts), do: data def decode( _query, From 72f76bf027785d0094ab1c7babdc69d34eb673e8 Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Fri, 6 Sep 2019 13:28:06 +0200 Subject: [PATCH 47/68] Rewrite Operation and add alter/3 function. --- lib/exdgraph.ex | 38 ++++++++++++++++++++-- lib/exdgraph/operation.ex | 68 +++++++++++++-------------------------- lib/exdgraph/protocol.ex | 13 +++----- 3 files changed, 64 insertions(+), 55 deletions(-) diff --git a/lib/exdgraph.ex b/lib/exdgraph.ex index 0449b95..910b35a 100755 --- a/lib/exdgraph.ex +++ b/lib/exdgraph.ex @@ -730,8 +730,42 @@ defmodule ExDgraph do %{:ok, %ExDgraph.Api.Payload{Data: ""}} """ - @spec operation(conn, String.t()) :: {:ok, ExDgraph.Response} | {:error, ExDgraph.Error} - defdelegate operation(conn, statement), to: Operation + # @spec operation(conn, String.t()) :: {:ok, ExDgraph.Result} | {:error, ExDgraph.Error} + # defdelegate operation(conn, statement), to: Operation + @spec alter(conn, iodata | map, Keyword.t()) :: {:ok, map} | {:error, ExDgraph.Error.t() | term} + def alter(conn, query, opts \\ []) + + def alter(conn, query, opts) when is_binary(query) do + operation = %Operation{schema: query} + + with {:ok, %Operation{} = operation, result} <- + DBConnection.prepare_execute(conn, operation, %{}, opts), + do: {:ok, query, result} + end + + @spec alter(conn, iodata | map, Keyword.t()) :: {:ok, map} | {:error, ExDgraph.Error.t() | term} + def alter(conn, query, opts) when is_map(query) do + operation = struct(Operation, query) + + with {:ok, %Operation{} = operation, result} <- + DBConnection.prepare_execute(conn, operation, %{}, opts), + do: {:ok, query, result} + end + + @doc """ + The same as `alter/3` but raises a ExDgraph.Exception if it fails. + Returns the server response otherwise. + """ + @spec alter!(conn, String.t()) :: ExDgraph.Payload | ExDgraph.Error + def alter!(conn, query) do + case alter(conn, query) do + {:ok, query, payload} -> + payload + + {:error, error} -> + raise error + end + end @doc """ The same as `operation/2` but raises an `ExDgraph.Exception` if it fails. diff --git a/lib/exdgraph/operation.ex b/lib/exdgraph/operation.ex index a69d0f7..00db327 100644 --- a/lib/exdgraph/operation.ex +++ b/lib/exdgraph/operation.ex @@ -2,58 +2,36 @@ defmodule ExDgraph.Operation do @moduledoc """ Provides the functions for the callbacks from the DBConnection behaviour. """ - alias ExDgraph.{Exception, OperationStatement} + alias ExDgraph.Error - @doc false - def operation(conn, operation) do - case operation_commit(conn, operation) do - {:error, f} -> - {:error, code: f.code, message: f.message} + defstruct drop_all: false, drop_attr: "", schema: "", txn_context: nil +end - {:ok, %OperationStatement{}, result} -> - {:ok, result} - end - end +defimpl DBConnection.Query, for: ExDgraph.Operation do + alias ExDgraph.{Operation, Payload} - @doc false - def operation!(conn, operation) do - case operation(conn, operation) do - {:ok, r} -> - r + def describe(query, _opts), do: query - {:error, code: code, message: message} -> - raise Exception, code: code, message: message - end + def parse(%{drop_attr: drop_attr, schema: schema, drop_all: _} = query, _opts) do + %Operation{ + query + | drop_attr: IO.iodata_to_binary(drop_attr), + schema: IO.iodata_to_binary(schema) + } end - defp operation_commit(conn, operation) do - operation_processed = - operation - |> Map.put_new(:drop_all, false) - |> Map.put_new(:drop_attr, "") - |> Map.put_new(:schema, "") - - exec = fn conn -> - operation = %OperationStatement{ - drop_all: operation_processed.drop_all, - drop_attr: operation_processed.drop_attr, - schema: operation_processed.schema - } - - case DBConnection.execute(conn, operation, %{}) do - {:ok, resp} -> - resp - - other -> - other - end - end - - # Response.transform(DBConnection.run(conn, exec, run_opts())) - DBConnection.run(conn, exec, run_opts()) + def encode(query, _, _) do + %{drop_all: drop_all, schema: schema, drop_attr: drop_attr} = query + %Operation{drop_all: drop_all, schema: schema, drop_attr: drop_attr} end - defp run_opts do - [pool: ExDgraph.config(:pool)] + def decode( + _query, + %ExDgraph.Api.Payload{Data: data} = _result, + _opts + ) do + %Payload{ + data: data + } end end diff --git a/lib/exdgraph/protocol.ex b/lib/exdgraph/protocol.ex index d918645..8c341c0 100755 --- a/lib/exdgraph/protocol.ex +++ b/lib/exdgraph/protocol.ex @@ -12,7 +12,7 @@ defmodule ExDgraph.Protocol do Error, Exception, MutationStatement, - OperationStatement, + Operation, Query } @@ -169,9 +169,9 @@ defmodule ExDgraph.Protocol do @impl true def handle_execute( - %OperationStatement{drop_all: drop_all, schema: schema, drop_attr: drop_attr} = query, + %Operation{drop_all: drop_all, schema: schema, drop_attr: drop_attr} = query, _params, - _, + _opts, %{channel: channel, opts: opts} = state ) do operation = Api.Operation.new(drop_all: drop_all, schema: schema, drop_attr: drop_attr) @@ -180,12 +180,9 @@ defmodule ExDgraph.Protocol do {:ok, res} -> {:ok, query, res, state} - {:error, f} -> - raise Exception, code: f.status, message: f.message + {:error, error} -> + {:error, %Error{action: :alter, code: error.status, reason: error.message}, state} end - rescue - e -> - {:error, e, state} end defp do_mutate(%{channel: channel, opts: opts} = state, dgraph_query, query) do From 3242d830381c185fe1222f19639c8524c1190835 Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Fri, 6 Sep 2019 13:29:28 +0200 Subject: [PATCH 48/68] Cleanup. --- lib/exdgraph.ex | 9 --------- 1 file changed, 9 deletions(-) diff --git a/lib/exdgraph.ex b/lib/exdgraph.ex index 910b35a..bd101f1 100755 --- a/lib/exdgraph.ex +++ b/lib/exdgraph.ex @@ -730,8 +730,6 @@ defmodule ExDgraph do %{:ok, %ExDgraph.Api.Payload{Data: ""}} """ - # @spec operation(conn, String.t()) :: {:ok, ExDgraph.Result} | {:error, ExDgraph.Error} - # defdelegate operation(conn, statement), to: Operation @spec alter(conn, iodata | map, Keyword.t()) :: {:ok, map} | {:error, ExDgraph.Error.t() | term} def alter(conn, query, opts \\ []) @@ -766,11 +764,4 @@ defmodule ExDgraph do raise error end end - - @doc """ - The same as `operation/2` but raises an `ExDgraph.Exception` if it fails. - Returns the server response otherwise. - """ - @spec operation!(conn, String.t()) :: ExDgraph.Response | ExDgraph.Exception - defdelegate operation!(conn, statement), to: Operation end From 6efdf07f12b6c3b153458d9fc8dd3ae0f01e4ba6 Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Fri, 6 Sep 2019 17:33:29 +0200 Subject: [PATCH 49/68] Cleanup and add typespecs. --- lib/exdgraph.ex | 17 +++++++------ lib/exdgraph/operation.ex | 50 ++++++++++++++++++++++++++------------- lib/exdgraph/query.ex | 41 +++++++++++++++++++++----------- lib/exdgraph/structs.ex | 3 ++- 4 files changed, 71 insertions(+), 40 deletions(-) diff --git a/lib/exdgraph.ex b/lib/exdgraph.ex index bd101f1..575d60e 100755 --- a/lib/exdgraph.ex +++ b/lib/exdgraph.ex @@ -344,7 +344,7 @@ defmodule ExDgraph do use Supervisor alias ExDgraph.Api.{Mutation, Operation} - alias ExDgraph.{Operation, Mutation, Protocol, Query, Utils} + alias ExDgraph.{Operation, Mutation, Protocol, Query} @type conn :: DBConnection.conn() # @type transaction :: DBConnection.t() @@ -354,7 +354,6 @@ defmodule ExDgraph do # Inherited from DBConnection @idle_timeout 5000 - @pool_timeout 5000 @timeout 15000 @doc """ @@ -505,7 +504,7 @@ defmodule ExDgraph do @spec query!(conn, String.t()) :: ExDgraph.Result | ExDgraph.Error def query!(conn, statement) do case query(conn, statement) do - {:ok, query, result} -> + {:ok, _query, result} -> result {:error, error} -> @@ -736,7 +735,7 @@ defmodule ExDgraph do def alter(conn, query, opts) when is_binary(query) do operation = %Operation{schema: query} - with {:ok, %Operation{} = operation, result} <- + with {:ok, %Operation{} = _operation, result} <- DBConnection.prepare_execute(conn, operation, %{}, opts), do: {:ok, query, result} end @@ -745,7 +744,7 @@ defmodule ExDgraph do def alter(conn, query, opts) when is_map(query) do operation = struct(Operation, query) - with {:ok, %Operation{} = operation, result} <- + with {:ok, %Operation{} = _operation, result} <- DBConnection.prepare_execute(conn, operation, %{}, opts), do: {:ok, query, result} end @@ -754,10 +753,10 @@ defmodule ExDgraph do The same as `alter/3` but raises a ExDgraph.Exception if it fails. Returns the server response otherwise. """ - @spec alter!(conn, String.t()) :: ExDgraph.Payload | ExDgraph.Error - def alter!(conn, query) do - case alter(conn, query) do - {:ok, query, payload} -> + @spec alter!(conn, String.t(), Keyword.t()) :: ExDgraph.Payload | ExDgraph.Error + def alter!(conn, query, opts \\ []) do + case alter(conn, query, opts) do + {:ok, _query, payload} -> payload {:error, error} -> diff --git a/lib/exdgraph/operation.ex b/lib/exdgraph/operation.ex index 00db327..92322c2 100644 --- a/lib/exdgraph/operation.ex +++ b/lib/exdgraph/operation.ex @@ -1,8 +1,14 @@ defmodule ExDgraph.Operation do @moduledoc """ - Provides the functions for the callbacks from the DBConnection behaviour. + Wrapper for operations sent to DBConnection. """ - alias ExDgraph.Error + + @type t :: %ExDgraph.Operation{ + drop_all: true | false | nil, + drop_attr: String.t(), + schema: String.t(), + txn_context: any() + } defstruct drop_all: false, drop_attr: "", schema: "", txn_context: nil end @@ -10,28 +16,40 @@ end defimpl DBConnection.Query, for: ExDgraph.Operation do alias ExDgraph.{Operation, Payload} - def describe(query, _opts), do: query - - def parse(%{drop_attr: drop_attr, schema: schema, drop_all: _} = query, _opts) do - %Operation{ - query - | drop_attr: IO.iodata_to_binary(drop_attr), - schema: IO.iodata_to_binary(schema) + @doc """ + This function is called to decode a result after it is returned by a connection callback module. + """ + def decode( + _query, + %ExDgraph.Api.Payload{Data: data} = _result, + _opts + ) do + %Payload{ + data: data } end + @doc """ + This function is called to describe a query after it is prepared using a connection callback module. + """ + def describe(query, _opts), do: query + + @doc """ + This function is called to encode a query before it is executed using a connection callback module. + """ def encode(query, _, _) do %{drop_all: drop_all, schema: schema, drop_attr: drop_attr} = query %Operation{drop_all: drop_all, schema: schema, drop_attr: drop_attr} end - def decode( - _query, - %ExDgraph.Api.Payload{Data: data} = _result, - _opts - ) do - %Payload{ - data: data + @doc """ + This function is called to parse a query term before it is prepared using a connection callback module. + """ + def parse(%{drop_attr: drop_attr, schema: schema, drop_all: _} = query, _opts) do + %Operation{ + query + | drop_attr: IO.iodata_to_binary(drop_attr), + schema: IO.iodata_to_binary(schema) } end end diff --git a/lib/exdgraph/query.ex b/lib/exdgraph/query.ex index 83a1e48..5fbefb4 100644 --- a/lib/exdgraph/query.ex +++ b/lib/exdgraph/query.ex @@ -1,8 +1,14 @@ defmodule ExDgraph.Query do @moduledoc """ - Provides the functions for the callbacks from the DBConnection behaviour. + Wrapper for queries sent to DBConnection. """ + @type t :: %ExDgraph.Query{ + statement: String.t(), + parameters: any(), + txn_context: any() + } + defstruct [:statement, :parameters, :txn_context] end @@ -13,21 +19,9 @@ defimpl DBConnection.Query, for: ExDgraph.Query do alias ExDgraph.{Query, Result, Utils} - def describe(query, _opts), do: query - @doc """ - Parse a query. - - This function is called to parse a query term before it is prepared. + This function is called to decode a result after it is returned by a connection callback module. """ - def parse(%{statement: nil} = query, _opts), do: %Query{query | statement: ""} - - def parse(%{statement: statement} = query, _opts) do - %Query{query | statement: IO.iodata_to_binary(statement)} - end - - def encode(_query, data, _opts), do: data - def decode( _query, %ExDgraph.Api.Response{json: json, schema: schema, txn: txn} = _result, @@ -44,4 +38,23 @@ defimpl DBConnection.Query, for: ExDgraph.Query do txn: txn } end + + @doc """ + This function is called to describe a query after it is prepared using a connection callback module. + """ + def describe(query, _opts), do: query + + @doc """ + This function is called to encode a query before it is executed using a connection callback module. + """ + def encode(_query, data, _opts), do: data + + @doc """ + This function is called to parse a query term before it is prepared using a connection callback module. + """ + def parse(%{statement: nil} = query, _opts), do: %Query{query | statement: ""} + + def parse(%{statement: statement} = query, _opts) do + %Query{query | statement: IO.iodata_to_binary(statement)} + end end diff --git a/lib/exdgraph/structs.ex b/lib/exdgraph/structs.ex index e034cce..cf29f90 100644 --- a/lib/exdgraph/structs.ex +++ b/lib/exdgraph/structs.ex @@ -2,7 +2,6 @@ defmodule ExDgraph.Error do @moduledoc """ Dgraph or connection error are wrapped in ExDgraph.Error. """ - defexception [:reason, :action, :code] @type t :: %ExDgraph.Error{ reason: String.t(), @@ -10,6 +9,8 @@ defmodule ExDgraph.Error do code: non_neg_integer } + defexception [:reason, :action, :code] + @impl true def message(%{action: action, reason: reason}) do "#{action} failed with #{inspect(reason)}" From 6534f4b2b00a1f916e37d123c1a78617dfae2d2a Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Fri, 6 Sep 2019 17:33:38 +0200 Subject: [PATCH 50/68] Upgrade credo. --- mix.exs | 2 +- mix.lock | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index 2dc18fa..dc3bcdb 100755 --- a/mix.exs +++ b/mix.exs @@ -43,7 +43,7 @@ defmodule ExDgraph.MixProject do {:jason, "~> 1.1"}, {:protobuf, "~> 0.6.1"}, {:excoveralls, "~> 0.10", only: :test}, - {:credo, "~> 1.0", only: [:dev, :test], runtime: false}, + {:credo, "~> 1.1.0", only: [:dev, :test], runtime: false}, {:ex_doc, "> 0.0.0", only: :dev, runtime: false}, {:mix_test_watch, "~> 0.5", only: :dev, runtime: false} ] diff --git a/mix.lock b/mix.lock index a78c22f..564bd83 100644 --- a/mix.lock +++ b/mix.lock @@ -6,8 +6,10 @@ "cowlib": {:hex, :cowlib, "2.6.0", "8aa629f81a0fc189f261dc98a42243fa842625feea3c7ec56c48f4ccdb55490f", [:rebar3], [], "hexpm"}, "credo": {:hex, :credo, "1.1.4", "c2f3b73c895d81d859cec7fcee7ffdb972c595fd8e85ab6f8c2adbf01cf7c29c", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, "db_connection": {:hex, :db_connection, "2.1.1", "a51e8a2ee54ef2ae6ec41a668c85787ed40cb8944928c191280fe34c15b76ae5", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm"}, + "dialyxir": {:hex, :dialyxir, "1.0.0-rc.6", "78e97d9c0ff1b5521dd68041193891aebebce52fc3b93463c0a6806874557d7d", [:mix], [{:erlex, "~> 0.2.1", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm"}, "earmark": {:hex, :earmark, "1.4.0", "397e750b879df18198afc66505ca87ecf6a96645545585899f6185178433cc09", [:mix], [], "hexpm"}, "elixir_uuid": {:hex, :elixir_uuid, "1.2.0", "ff26e938f95830b1db152cb6e594d711c10c02c6391236900ddd070a6b01271d", [:mix], [], "hexpm"}, + "erlex": {:hex, :erlex, "0.2.4", "23791959df45fe8f01f388c6f7eb733cc361668cbeedd801bf491c55a029917b", [:mix], [], "hexpm"}, "ex_doc": {:hex, :ex_doc, "0.21.2", "caca5bc28ed7b3bdc0b662f8afe2bee1eedb5c3cf7b322feeeb7c6ebbde089d6", [:mix], [{:earmark, "~> 1.3.3 or ~> 1.4", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm"}, "excoveralls": {:hex, :excoveralls, "0.11.2", "0c6f2c8db7683b0caa9d490fb8125709c54580b4255ffa7ad35f3264b075a643", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, "file_system": {:hex, :file_system, "0.2.7", "e6f7f155970975789f26e77b8b8d8ab084c59844d8ecfaf58cbda31c494d14aa", [:mix], [], "hexpm"}, From b1faa11f502c530859dae0465dda97f7dcbd0434 Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Fri, 6 Sep 2019 17:41:22 +0200 Subject: [PATCH 51/68] Implement missing DBConnection callbacks. --- lib/exdgraph/protocol.ex | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/lib/exdgraph/protocol.ex b/lib/exdgraph/protocol.ex index 8c341c0..399e58f 100755 --- a/lib/exdgraph/protocol.ex +++ b/lib/exdgraph/protocol.ex @@ -185,6 +185,26 @@ defmodule ExDgraph.Protocol do end end + @impl true + def handle_close(_query, _opts, state) do + {:ok, nil, state} + end + + @impl true + def handle_deallocate(_query, _cursor, _opts, state) do + {:ok, nil, state} + end + + @impl true + def handle_declare(query, _params, _opts, state) do + {:ok, query, nil, state} + end + + @impl true + def handle_fetch(_query, _cursor, _opts, state) do + {:halt, nil, state} + end + defp do_mutate(%{channel: channel, opts: opts} = state, dgraph_query, query) do case ExDgraph.Api.Dgraph.Stub.mutate(channel, dgraph_query, timeout: opts[:timeout]) do {:ok, res} -> From e01ad16f2b82cc00a112c5e313a53d00c5fa753a Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Fri, 6 Sep 2019 17:54:50 +0200 Subject: [PATCH 52/68] Move query tests to separate file. --- test/exdgraph_test.exs | 62 ---------------------------------------- test/query_test.exs | 65 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 62 deletions(-) create mode 100644 test/query_test.exs diff --git a/test/exdgraph_test.exs b/test/exdgraph_test.exs index 3ed4701..bcdc1fa 100755 --- a/test/exdgraph_test.exs +++ b/test/exdgraph_test.exs @@ -3,16 +3,6 @@ defmodule ExDgraphTest do import ExDgraph.TestHelper import ExUnit.CaptureLog - alias ExDgraph.{Error, Query, Result} - - setup_all do - {:ok, conn} = ExDgraph.start_link() - ExDgraph.operation(conn, %{drop_all: true}) - import_starwars_sample(conn) - - [conn: conn] - end - describe "start_link/1" do test "it starts the client" do {status, pid} = ExDgraph.start_link() @@ -49,56 +39,4 @@ defmodule ExDgraphTest do "** (ExDgraph.Error) connect failed with \"Error when opening connection: :timeout\"" end end - - describe "Query" do - @sample_query """ - { - starwars(func: anyofterms(name, "VI")) - { - uid - name - release_date - starring - { - name - } - } - } - """ - - test "query/3 with correct query returns {:ok, %Query{}, %Result{}}", %{conn: conn} do - {status, %Query{} = _query, %Result{} = result} = ExDgraph.query(conn, @sample_query) - assert status == :ok - data = result.data - starwars = List.first(data.starwars) - assert starwars.name == "Star Wars: Episode VI - Return of the Jedi" - assert starwars.release_date == "1983-05-25" - assert length(starwars.starring) == 3 - end - - test "query/3 with wrong query returns {:error, %Error{}}", %{conn: conn} do - {status, %Error{} = error} = ExDgraph.query(conn, "wrong") - assert status == :error - - assert %Error{} = error - assert error.code == 2 - assert error.action == :query - assert error.reason =~ "Invalid operation type: wrong" - end - - test "query!/2 with correct query returns query_msg", %{conn: conn} do - %Result{} = result = ExDgraph.query!(conn, @sample_query) - data = result.data - starwars = List.first(data.starwars) - assert starwars.name == "Star Wars: Episode VI - Return of the Jedi" - assert starwars.release_date == "1983-05-25" - assert length(starwars.starring) == 3 - end - - test "query!/2 raises ExDgraph.Exception", %{conn: conn} do - assert_raise ExDgraph.Error, fn -> - ExDgraph.query!(conn, "wrong") - end - end - end end diff --git a/test/query_test.exs b/test/query_test.exs new file mode 100644 index 0000000..70be5ce --- /dev/null +++ b/test/query_test.exs @@ -0,0 +1,65 @@ +defmodule ExDgraph.QueryTest do + @moduledoc false + + use ExUnit.Case + import ExDgraph.TestHelper + alias ExDgraph.{Error, Query, Result} + + @sample_query """ + { + starwars(func: anyofterms(name, "VI")) + { + uid + name + release_date + starring + { + name + } + } + } + """ + + setup_all do + {:ok, conn} = ExDgraph.start_link() + ExDgraph.alter(conn, %{drop_all: true}) + import_starwars_sample(conn) + + [conn: conn] + end + + test "query/3 with correct query returns {:ok, %Query{}, %Result{}}", %{conn: conn} do + {status, %Query{} = _query, %Result{} = result} = ExDgraph.query(conn, @sample_query) + assert status == :ok + data = result.data + starwars = List.first(data.starwars) + assert starwars.name == "Star Wars: Episode VI - Return of the Jedi" + assert starwars.release_date == "1983-05-25" + assert length(starwars.starring) == 3 + end + + test "query/3 with wrong query returns {:error, %Error{}}", %{conn: conn} do + {status, %Error{} = error} = ExDgraph.query(conn, "wrong") + assert status == :error + + assert %Error{} = error + assert error.code == 2 + assert error.action == :query + assert error.reason =~ "Invalid operation type: wrong" + end + + test "query!/2 with correct query returns query_msg", %{conn: conn} do + %Result{} = result = ExDgraph.query!(conn, @sample_query) + data = result.data + starwars = List.first(data.starwars) + assert starwars.name == "Star Wars: Episode VI - Return of the Jedi" + assert starwars.release_date == "1983-05-25" + assert length(starwars.starring) == 3 + end + + test "query!/2 raises ExDgraph.Exception", %{conn: conn} do + assert_raise ExDgraph.Error, fn -> + ExDgraph.query!(conn, "wrong") + end + end +end From 269350e7c1c0abcf327568a24cfb758ca95fbbe5 Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Fri, 6 Sep 2019 17:55:37 +0200 Subject: [PATCH 53/68] Adapt TestHelper to removal of ExDgraph.operation/2. --- test/test_helper.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_helper.exs b/test/test_helper.exs index ef0e2e9..041c4f8 100755 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -61,7 +61,7 @@ defmodule ExDgraph.TestHelper do """ def import_starwars_sample(conn) do - ExDgraph.operation(conn, %{schema: @starwars_schema}) + ExDgraph.alter(conn, %{schema: @starwars_schema}) {:ok, _} = ExDgraph.mutation(conn, @starwars_creation_mutation) end @@ -71,7 +71,7 @@ defmodule ExDgraph.TestHelper do def drop_all() do conn = ExDgraph.conn() - ExDgraph.operation(conn, %{drop_all: true}) + ExDgraph.alter(conn, %{drop_all: true}) end end From 837cd51e34e0dc2ff070b7d376aa60ec1c9aba2b Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Fri, 6 Sep 2019 17:55:47 +0200 Subject: [PATCH 54/68] Fix operation tests. --- test/operation_test.exs | 46 ++++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/test/operation_test.exs b/test/operation_test.exs index e5565a1..dec57ba 100644 --- a/test/operation_test.exs +++ b/test/operation_test.exs @@ -1,43 +1,47 @@ -defmodule OperationTest do +defmodule ExDgraph.OperationTest do @moduledoc """ """ use ExUnit.Case require Logger import ExDgraph.TestHelper - setup do - conn = ExDgraph.conn() - drop_all() - import_starwars_sample() + alias ExDgraph.{Error, Payload} - on_exit(fn -> - # close channel ? - :ok - end) + setup do + {:ok, conn} = ExDgraph.start_link() + ExDgraph.alter(conn, %{drop_all: true}) + import_starwars_sample(conn) [conn: conn] end - test "operation(%{drop_all: true}) is successful", %{conn: conn} do - {status, operation_msg} = ExDgraph.operation(conn, %{drop_all: true}) + test "alter(%{drop_all: true}) is successful", %{conn: conn} do + {status, _operation, payload} = ExDgraph.alter(conn, %{drop_all: true}) assert status == :ok - assert operation_msg == %ExDgraph.Api.Payload{Data: ""} + assert payload == %Payload{data: ""} end - test "operation/2 returns {:error, error} for incorrect operation", %{conn: conn} do - {status, error} = ExDgraph.operation(conn, %{}) + test "alter/3 returns the operation", %{conn: conn} do + {:ok, operation, _payload} = ExDgraph.alter(conn, %{drop_all: true}) + assert %{drop_all: true} = operation + end + + test "alter/3 returns {:error, error} for incorrect operation", %{conn: conn} do + {status, error} = ExDgraph.alter(conn, %{}) assert status == :error - assert error[:code] == 2 + + assert %Error{action: :alter, code: 2, reason: "Operation must have at least one field set"} = + error end - test "operation!(%{drop_all: true}) is successful", %{conn: conn} do - operation_msg = ExDgraph.operation!(conn, %{drop_all: true}) - assert operation_msg == %ExDgraph.Api.Payload{Data: ""} + test "alter!(%{drop_all: true}) is successful", %{conn: conn} do + payload = ExDgraph.alter!(conn, %{drop_all: true}) + assert payload == %Payload{data: ""} end - test "operation!/2 raises ExDgraph.Exception for incorrect operation", %{conn: conn} do - assert_raise ExDgraph.Exception, fn -> - ExDgraph.operation!(conn, %{}) + test "alter!/3 raises ExDgraph.Exception for incorrect operation", %{conn: conn} do + assert_raise Error, fn -> + ExDgraph.alter!(conn, %{}) end end end From bf5068aa86eacc6c7aca658477928d13fd30311b Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Fri, 6 Sep 2019 18:11:18 +0200 Subject: [PATCH 55/68] Add additional tests. --- test/operation_test.exs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/operation_test.exs b/test/operation_test.exs index dec57ba..b78f450 100644 --- a/test/operation_test.exs +++ b/test/operation_test.exs @@ -21,6 +21,18 @@ defmodule ExDgraph.OperationTest do assert payload == %Payload{data: ""} end + test "alter(%{drop_attr: attr}) is successful", %{conn: conn} do + {status, _operation, payload} = ExDgraph.alter(conn, %{drop_attr: "name"}) + assert status == :ok + assert payload == %Payload{data: ""} + end + + test "alter(%{schema: schema}) is successful", %{conn: conn} do + {status, _operation, payload} = ExDgraph.alter(conn, %{schema: "name: string @index(term) ."}) + assert status == :ok + assert payload == %Payload{data: ""} + end + test "alter/3 returns the operation", %{conn: conn} do {:ok, operation, _payload} = ExDgraph.alter(conn, %{drop_all: true}) assert %{drop_all: true} = operation From 667cde39f2b13359ba5ef1e5aaaf4d25f540c2f0 Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Fri, 6 Sep 2019 18:11:29 +0200 Subject: [PATCH 56/68] Formatting. --- lib/exdgraph.ex | 4 ++-- test/exdgraph_test.exs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/exdgraph.ex b/lib/exdgraph.ex index 575d60e..3156da0 100755 --- a/lib/exdgraph.ex +++ b/lib/exdgraph.ex @@ -353,8 +353,8 @@ defmodule ExDgraph do # Inherited from DBConnection - @idle_timeout 5000 - @timeout 15000 + @idle_timeout 5_000 + @timeout 15_000 @doc """ Start the connection process and connect to Dgraph diff --git a/test/exdgraph_test.exs b/test/exdgraph_test.exs index bcdc1fa..8b0b8a0 100755 --- a/test/exdgraph_test.exs +++ b/test/exdgraph_test.exs @@ -27,14 +27,14 @@ defmodule ExDgraphTest do assert capture_log(fn -> {:ok, pid} = ExDgraph.start_link([hostname: "non_existing"] ++ opts) - assert_receive {:EXIT, ^pid, :killed}, 10000 + assert_receive {:EXIT, ^pid, :killed}, 10_000 end) =~ "** (ExDgraph.Error) connect failed with \"Error when opening connection: :timeout\"" assert capture_log(fn -> {:ok, pid} = ExDgraph.start_link([port: 700] ++ opts) - assert_receive {:EXIT, ^pid, :killed}, 10000 + assert_receive {:EXIT, ^pid, :killed}, 10_000 end) =~ "** (ExDgraph.Error) connect failed with \"Error when opening connection: :timeout\"" end From fac80fa7eb824d34387a8e95d4420c058b6bd640 Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Tue, 10 Sep 2019 22:20:16 +0200 Subject: [PATCH 57/68] Rename Result -> QueryResult. --- lib/exdgraph.ex | 4 ++-- lib/exdgraph/query.ex | 4 ++-- lib/exdgraph/structs.ex | 10 ++++++---- test/query_test.exs | 6 +++--- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/lib/exdgraph.ex b/lib/exdgraph.ex index 3156da0..dbcd5f2 100755 --- a/lib/exdgraph.ex +++ b/lib/exdgraph.ex @@ -343,7 +343,7 @@ defmodule ExDgraph do use Supervisor - alias ExDgraph.Api.{Mutation, Operation} + # alias ExDgraph.Api alias ExDgraph.{Operation, Mutation, Protocol, Query} @type conn :: DBConnection.conn() @@ -501,7 +501,7 @@ defmodule ExDgraph do The same as `query/3` but raises a ExDgraph.Exception if it fails. Returns the server response otherwise. """ - @spec query!(conn, String.t()) :: ExDgraph.Result | ExDgraph.Error + @spec query!(conn, String.t()) :: ExDgraph.QueryResult | ExDgraph.Error def query!(conn, statement) do case query(conn, statement) do {:ok, _query, result} -> diff --git a/lib/exdgraph/query.ex b/lib/exdgraph/query.ex index 5fbefb4..95ac4e9 100644 --- a/lib/exdgraph/query.ex +++ b/lib/exdgraph/query.ex @@ -17,7 +17,7 @@ defimpl DBConnection.Query, for: ExDgraph.Query do Implementation of `DBConnection.Query` protocol. """ - alias ExDgraph.{Query, Result, Utils} + alias ExDgraph.{Api, Query, QueryResult, Utils} @doc """ This function is called to decode a result after it is returned by a connection callback module. @@ -32,7 +32,7 @@ defimpl DBConnection.Query, for: ExDgraph.Query do |> Jason.decode!() |> Utils.atomify_map_keys() - %Result{ + %QueryResult{ data: data, schema: schema, txn: txn diff --git a/lib/exdgraph/structs.ex b/lib/exdgraph/structs.ex index cf29f90..77a08e7 100644 --- a/lib/exdgraph/structs.ex +++ b/lib/exdgraph/structs.ex @@ -17,18 +17,20 @@ defmodule ExDgraph.Error do end end -defmodule ExDgraph.Result do +defmodule ExDgraph.QueryResult do @moduledoc """ - Results from a query are wrapped in ExDgraph.Result + Results from a query are wrapped in ExDgraph.QueryResult """ @type t :: %__MODULE__{ data: %{optional(any) => any}, schema: [any()], - txn: %ExDgraph.Api.TxnContext{} + txn: %ExDgraph.Api.TxnContext{}, + uids: map() | nil } - defstruct [:data, :schema, :txn] + defstruct [:data, :schema, :txn, :uids] +end end defmodule ExDgraph.Payload do diff --git a/test/query_test.exs b/test/query_test.exs index 70be5ce..56480f8 100644 --- a/test/query_test.exs +++ b/test/query_test.exs @@ -3,7 +3,7 @@ defmodule ExDgraph.QueryTest do use ExUnit.Case import ExDgraph.TestHelper - alias ExDgraph.{Error, Query, Result} + alias ExDgraph.{Error, Query, QueryResult} @sample_query """ { @@ -29,7 +29,7 @@ defmodule ExDgraph.QueryTest do end test "query/3 with correct query returns {:ok, %Query{}, %Result{}}", %{conn: conn} do - {status, %Query{} = _query, %Result{} = result} = ExDgraph.query(conn, @sample_query) + {status, %Query{} = _query, %QueryResult{} = result} = ExDgraph.query(conn, @sample_query) assert status == :ok data = result.data starwars = List.first(data.starwars) @@ -49,7 +49,7 @@ defmodule ExDgraph.QueryTest do end test "query!/2 with correct query returns query_msg", %{conn: conn} do - %Result{} = result = ExDgraph.query!(conn, @sample_query) + %QueryResult{} = result = ExDgraph.query!(conn, @sample_query) data = result.data starwars = List.first(data.starwars) assert starwars.name == "Star Wars: Episode VI - Return of the Jedi" From b846b31b71a42bad0d7be04437366a2703c8c785 Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Tue, 10 Sep 2019 22:26:35 +0200 Subject: [PATCH 58/68] Pass in opts to query!/3 --- lib/exdgraph.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/exdgraph.ex b/lib/exdgraph.ex index dbcd5f2..7cd431a 100755 --- a/lib/exdgraph.ex +++ b/lib/exdgraph.ex @@ -498,12 +498,12 @@ defmodule ExDgraph do end @doc """ - The same as `query/3` but raises a ExDgraph.Exception if it fails. + The same as `query/3` but raises a ExDgraph.Error if it fails. Returns the server response otherwise. """ @spec query!(conn, String.t()) :: ExDgraph.QueryResult | ExDgraph.Error - def query!(conn, statement) do - case query(conn, statement) do + def query!(conn, statement, opts \\ []) do + case query(conn, statement, opts) do {:ok, _query, result} -> result From f8de15fd181d2ee877ce1de9e267d9f5c70a35b7 Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Tue, 10 Sep 2019 22:27:15 +0200 Subject: [PATCH 59/68] Rewrite mutations. --- lib/exdgraph.ex | 54 +++++--- lib/exdgraph/mutation.ex | 259 ++++++++++++++------------------------ lib/exdgraph/protocol.ex | 30 +---- lib/exdgraph/structs.ex | 16 +++ lib/exdgraph/transform.ex | 15 --- test/test_helper.exs | 2 +- 6 files changed, 154 insertions(+), 222 deletions(-) delete mode 100644 lib/exdgraph/transform.ex diff --git a/lib/exdgraph.ex b/lib/exdgraph.ex index 7cd431a..33a65bd 100755 --- a/lib/exdgraph.ex +++ b/lib/exdgraph.ex @@ -516,7 +516,7 @@ defmodule ExDgraph do ###################### @doc """ - Sends the mutation to the server and returns `{:ok, result}` or + Sends the mutation to the server and returns `{:ok, query, result}` or `{:error, error}` otherwise ## Examples @@ -564,8 +564,9 @@ defmodule ExDgraph do \"\"\" ``` - iex> ExDgraph.mutation(conn, starwars_creation_mutation) + iex> ExDgraph.mutate(conn, starwars_creation_mutation) %{:ok, + query, %{ context: %ExDgraph.Api.TxnContext{ aborted: false, @@ -590,15 +591,40 @@ defmodule ExDgraph do } """ - @spec mutation(conn, String.t()) :: {:ok, ExDgraph.Result} | {:error, ExDgraph.Error} - defdelegate mutation(conn, statement), to: Mutation + @spec mutate(conn, iodata | map() | struct(), Keyword.t()) :: + {:ok, map()} | {:ok, ExDgraph.MutationResult} | {:error, ExDgraph.Error.t() | term} + def mutate(conn, query, opts \\ []) + + def mutate(conn, query, opts) do + mutation = %Mutation{statement: query} + + with {:ok, %Mutation{} = _query, result} <- + DBConnection.prepare_execute(conn, mutation, %{}, opts), + do: {:ok, query, result} + end + + @doc """ + The same as `mutate/3` but raises a ExDgraph.Error if it fails. + Returns the server response otherwise. + """ + @spec mutate(conn, iodata | map() | struct(), Keyword.t()) :: + ExDgraph.MutationResult | ExDgraph.Error.t() + def mutate!(conn, statement, opts \\ []) do + case mutate(conn, statement, opts) do + {:ok, _query, result} -> + result + + {:error, error} -> + raise error + end + end @doc """ The same as `mutation/2` but raises an `ExDgraph.Exception` if it fails. Returns the server response otherwise. """ - @spec mutation!(conn, String.t()) :: ExDgraph.Result | ExDgraph.Exception - defdelegate mutation!(conn, statement), to: Mutation + # @spec mutation!(conn, String.t()) :: ExDgraph.MutationResult | ExDgraph.Exception + # defdelegate mutation!(conn, statement), to: Mutation @doc """ Allow you to pass a map to insert into the database. The function sends the mutation to the server and returns `{:ok, result}` or `{:error, error}` otherwise. Internally it uses Dgraphs `set_json`. @@ -637,15 +663,15 @@ defmodule ExDgraph do } } """ - @spec set_map(conn, Map.t()) :: {:ok, ExDgraph.Result} | {:error, ExDgraph.Error} - defdelegate set_map(conn, map), to: Mutation + # @spec set_map(conn, Map.t()) :: {:ok, ExDgraph.Result} | {:error, ExDgraph.Error} + # defdelegate set_map(conn, map), to: Mutation @doc """ The same as `set_map/2` but raises an `ExDgraph.Exception` if it fails. Returns the server response otherwise. """ - @spec set_map!(conn, Map.t()) :: {:ok, ExDgraph.Result} | {:error, ExDgraph.Error} - defdelegate set_map!(conn, map), to: Mutation + # @spec set_map!(conn, Map.t()) :: {:ok, ExDgraph.Result} | {:error, ExDgraph.Error} + # defdelegate set_map!(conn, map), to: Mutation @doc """ This function allow you to convert an struct type into a mutation (json based) type in dgraph. @@ -690,15 +716,15 @@ defmodule ExDgraph do } } """ - @spec set_struct(conn, Map.t()) :: {:ok, ExDgraph.Result} | {:error, ExDgraph.Error} - defdelegate set_struct(conn, map), to: Mutation + # @spec set_struct(conn, Map.t()) :: {:ok, ExDgraph.Result} | {:error, ExDgraph.Error} + # defdelegate set_struct(conn, map), to: Mutation @doc """ The same as `set_struct/2` but raises an `ExDgraph.Exception` if it fails. Returns the server response otherwise. """ - @spec set_struct!(conn, Map.t()) :: {:ok, ExDgraph.Result} | {:error, ExDgraph.Error} - defdelegate set_struct!(conn, map), to: Mutation + # @spec set_struct!(conn, Map.t()) :: {:ok, ExDgraph.Result} | {:error, ExDgraph.Error} + # defdelegate set_struct!(conn, map), to: Mutation ## Operation ###################### diff --git a/lib/exdgraph/mutation.ex b/lib/exdgraph/mutation.ex index 2676e3c..1d86e94 100644 --- a/lib/exdgraph/mutation.ex +++ b/lib/exdgraph/mutation.ex @@ -1,167 +1,122 @@ defmodule ExDgraph.Mutation do @moduledoc """ - Provides the functions for the callbacks from the DBConnection behaviour. + Wrapper for mutations sent to DBConnection. """ - alias ExDgraph.{Exception, MutationStatement, Transform} - @doc false - def mutation(conn, statement) do - case mutation_commit(conn, statement) do - {:ok, %MutationStatement{}, result} -> - {:ok, result} + alias ExDgraph.{Exception, Transform} - {:error, f} -> - {:error, code: f.code, message: f.message} - end - end - - @doc false - def mutation!(conn, statement) do - case mutation(conn, statement) do - {:ok, %MutationStatement{}, result} -> - {:ok, result} - - {:error, code: code, message: message} -> - raise Exception, code: code, message: message - end - end - - @doc false - def set_map(conn, map) do - map_with_tmp_uids = insert_tmp_uids(map) - json = Jason.encode!(map_with_tmp_uids) - - case set_map_commit(conn, json, map_with_tmp_uids) do - {:ok, r} -> - {:ok, r} - - {:error, f} -> - {:error, code: f.code, message: f.message} - end - end - - @doc false - def set_map!(conn, map) do - case set_map(conn, map) do - {:ok, r} -> - r - - {:error, code: code, message: message} -> - raise Exception, code: code, message: message - end - end + @type t :: %ExDgraph.Mutation{ + statement: String.t() | map() | struct(), + set_json: String.t(), + txn_context: any(), + original_statement: any() + } - @doc false - def set_struct(conn, struct) do - uids_and_schema_map = set_tmp_ids_and_schema(struct) - json = Jason.encode!(uids_and_schema_map) - - case set_struct_commit(conn, json, uids_and_schema_map) do - {:ok, r} -> - {:ok, r} + defstruct statement: nil, set_json: nil, txn_context: nil, original_statement: nil +end - {:error, f} -> - {:error, code: f.code, message: f.message} - end - end +defimpl DBConnection.Query, for: ExDgraph.Mutation do + @moduledoc """ + Implementation of `DBConnection.Query` protocol. + """ - @doc false - def set_struct!(conn, struct) do - case set_struct(conn, struct) do - {:ok, r} -> - r + alias ExDgraph.{Api, Mutation, MutationResult, Transform, Utils} - {:error, code: code, message: message} -> - raise Exception, code: code, message: message - end + @doc """ + This function is called to decode a result after it is returned by a connection callback module. + """ + def decode( + %Mutation{ + set_json: set_json, + statement: statement, + original_statement: original_statement + } = query, + %Api.Assigned{context: context, latency: latency, uids: uids} = _result, + _opts + ) + when is_binary(set_json) and is_nil(statement) do + data = + set_json + |> Jason.decode!() + |> Utils.atomify_map_keys() + |> replace_tmp_uids(uids) + |> merge_result_with_statement(original_statement) + + %MutationResult{ + data: data, + context: context, + latency: latency, + uids: uids + } end - defp mutation_commit(conn, statement) do - exec = fn conn -> - q = %MutationStatement{statement: statement} - - case DBConnection.execute(conn, q, %{}) do - {:ok, resp} -> Transform.transform_mutation(resp) - other -> other - end - end - - DBConnection.run(conn, exec) + def decode( + %Mutation{ + set_json: set_json, + statement: statement + } = query, + %Api.Assigned{context: context, latency: latency, uids: uids} = _result, + _opts + ) + when is_nil(set_json) and is_binary(statement) do + %MutationResult{ + context: context, + latency: latency, + uids: uids + } end - defp set_map_commit(conn, json, map_with_tmp_uids) do - exec = fn conn -> - q = %MutationStatement{set_json: json} - - case DBConnection.execute(conn, q, %{}) do - {:ok, %MutationStatement{}, result} -> - # Now exchange the tmp ids for the ones returned from the db - result_with_uids = replace_tmp_uids(map_with_tmp_uids, result.uids) - {:ok, Map.put(result, :result, result_with_uids)} + @doc """ + This function is called to describe a query after it is prepared using a connection callback module. + """ + def describe(query, _opts), do: query - other -> - other - end - end + @doc """ + This function is called to encode a query before it is executed using a connection callback module. + """ + def encode(_query, data, _opts), do: data - DBConnection.run(conn, exec) + @doc """ + This function is called to parse a query term before it is prepared using a connection callback module. + """ + def parse(%{statement: %{__struct__: struct_name} = statement} = query, _opts) do + json = + statement + |> Map.from_struct() + |> insert_tmp_uids() + |> Jason.encode!() + + %Mutation{query | statement: nil, set_json: json, original_statement: statement} end - defp set_struct_commit(conn, json, struct_with_tmp_uids) do - exec = fn conn -> - q = %MutationStatement{set_json: json} - - case DBConnection.execute(conn, q, %{}) do - {:ok, %MutationStatement{}, result} -> - # Now exchange the tmp ids for the ones returned from the db - result_with_uids = replace_tmp_struct_uids(struct_with_tmp_uids, result.uids) - {:ok, Map.put(result, :result, result_with_uids)} - - other -> - other - end - end + def parse(%{statement: statement} = query, _opts) when is_map(statement) do + json = + statement + |> insert_tmp_uids() + |> Jason.encode!() - DBConnection.run(conn, exec) + %Mutation{query | statement: nil, set_json: json, original_statement: statement} end - defp insert_tmp_uids(map) when is_list(map), do: Enum.map(map, &insert_tmp_uids/1) - - defp insert_tmp_uids(map) when is_map(map) do - map - |> Map.update(:uid, "_:#{UUID.uuid4()}", fn existing_uuid -> existing_uuid end) - |> Enum.reduce(%{}, fn {key, map_value}, a -> - Map.merge(a, %{key => insert_tmp_uids(map_value)}) - end) + def parse(%{statement: statement} = query, _opts) when is_binary(statement) do + %Mutation{query | statement: IO.iodata_to_binary(statement), original_statement: statement} end - defp insert_tmp_uids(value), do: value - - defp set_tmp_ids_and_schema(map) when is_list(map), do: Enum.map(map, &set_tmp_ids_and_schema/1) - - defp set_tmp_ids_and_schema(%x{} = map) do - schema = x |> get_schema_name() + defp insert_tmp_uids(map, acc \\ 0) - map - |> Map.from_struct() - |> Map.update(:uid, "_:#{UUID.uuid4()}", fn - nil -> "_:#{UUID.uuid4()}" - existing_uuid -> existing_uuid - end) - |> Enum.reduce(%{}, fn {key, map_value}, a -> - set_schema(schema, {key, map_value}, a, ExDgraph.config(:enforce_struct_schema)) - end) - end + defp insert_tmp_uids(map, acc) when is_list(map), + do: Enum.map(map, &insert_tmp_uids(&1, acc).()) - defp set_tmp_ids_and_schema(map) when is_map(map) do + defp insert_tmp_uids(map, acc) when is_map(map) do map - |> Map.update(:uid, "_:#{UUID.uuid4()}", fn existing_uuid -> existing_uuid end) + |> Map.update(:uid, "_:#{acc}", fn existing_uuid -> existing_uuid end) |> Enum.reduce(%{}, fn {key, map_value}, a -> - Map.merge(a, %{key => set_tmp_ids_and_schema(map_value)}) + new_acc = acc + 1 + Map.merge(a, %{key => insert_tmp_uids(map_value, new_acc)}) end) end - defp set_tmp_ids_and_schema(value), do: value + defp insert_tmp_uids(value, acc), do: value defp replace_tmp_uids(map, uids) when is_list(map), do: Enum.map(map, &replace_tmp_uids(&1, uids)) @@ -181,37 +136,9 @@ defmodule ExDgraph.Mutation do defp replace_tmp_uids(value, _uids), do: value - defp replace_tmp_struct_uids(map, uids) when is_list(map), - do: Enum.map(map, &replace_tmp_struct_uids(&1, uids)) - - defp replace_tmp_struct_uids(map, uids) when is_map(map) do - map - |> Map.update(:uid, map[:uid], fn existing_uuid -> - case String.slice(existing_uuid, 0, 2) == "_:" do - true -> uids[String.replace_leading(existing_uuid, "_:", "")] - false -> existing_uuid - end - end) - |> Enum.reduce(%{}, fn {key, map_value}, a -> - # delete the schema prefix - key = key |> to_string() |> String.split(".") |> List.last() |> String.to_existing_atom() - Map.merge(a, %{key => replace_tmp_struct_uids(map_value, uids)}) - end) - end - - defp replace_tmp_struct_uids(value, _uids), do: value - - defp get_schema_name(schema) do - schema |> to_string() |> String.split(".") |> List.last() |> String.downcase() + defp merge_result_with_statement(result, %{__struct__: struct_name} = _original_statement) do + struct(struct_name, result) end - defp set_schema(_schema_name, {:uid, map_value}, result, _is_enforced_schema), - do: Map.merge(result, %{:uid => set_tmp_ids_and_schema(map_value)}) - - defp set_schema(schema_name, {key, map_value}, result, is_enforced_schema) - when is_enforced_schema == true, - do: Map.merge(result, %{"#{schema_name}.#{key}" => set_tmp_ids_and_schema(map_value)}) - - defp set_schema(_schema_name, {key, map_value}, result, _is_enforced_schema), - do: Map.merge(result, %{key => set_tmp_ids_and_schema(map_value)}) + defp merge_result_with_statement(result, _original_statement), do: result end diff --git a/lib/exdgraph/protocol.ex b/lib/exdgraph/protocol.ex index 399e58f..ef142c0 100755 --- a/lib/exdgraph/protocol.ex +++ b/lib/exdgraph/protocol.ex @@ -11,7 +11,7 @@ defmodule ExDgraph.Protocol do Api, Error, Exception, - MutationStatement, + Mutation, Operation, Query } @@ -134,39 +134,17 @@ defmodule ExDgraph.Protocol do @impl true def handle_execute( - %MutationStatement{statement: statement, set_json: ""} = query, + %Mutation{statement: statement, set_json: set_json} = query, _params, _, state ) do - dgraph_query = ExDgraph.Api.Mutation.new(set_nquads: statement, commit_now: true) - do_mutate(state, dgraph_query, query) - end + dgraph_query = + ExDgraph.Api.Mutation.new(set_nquads: statement, set_json: set_json, commit_now: true) - @impl true - def handle_execute( - %MutationStatement{statement: "", set_json: set_json} = query, - _request, - _opts, - state - ) do - dgraph_query = ExDgraph.Api.Mutation.new(set_json: set_json, commit_now: true) do_mutate(state, dgraph_query, query) end - @impl true - def handle_execute( - %MutationStatement{statement: _statement, set_json: _set_json}, - _params, - _, - %{channel: channel} = state - ) do - raise Exception, code: 2, message: "Both set_json and statement defined" - rescue - e -> - {:error, e, state} - end - @impl true def handle_execute( %Operation{drop_all: drop_all, schema: schema, drop_attr: drop_attr} = query, diff --git a/lib/exdgraph/structs.ex b/lib/exdgraph/structs.ex index 77a08e7..a245eaf 100644 --- a/lib/exdgraph/structs.ex +++ b/lib/exdgraph/structs.ex @@ -31,6 +31,22 @@ defmodule ExDgraph.QueryResult do defstruct [:data, :schema, :txn, :uids] end + +defmodule ExDgraph.MutationResult do + @moduledoc """ + Results from a mutation are wrapped in ExDgraph.MutationResult + """ + + alias ExDgraph.Api + + @type t :: %__MODULE__{ + data: %{optional(any) => any}, + uids: %{String.t() => String.t()}, + context: Api.TxnContext.t() | nil, + latency: Api.Latency.t() | nil + } + + defstruct [:data, :uids, :context, :latency] end defmodule ExDgraph.Payload do diff --git a/lib/exdgraph/transform.ex b/lib/exdgraph/transform.ex deleted file mode 100644 index 350f7f8..0000000 --- a/lib/exdgraph/transform.ex +++ /dev/null @@ -1,15 +0,0 @@ -defmodule ExDgraph.Transform do - @moduledoc """ - Transform a raw response from Dgraph. For example decode the json part. - """ - - @doc """ - Takes a response from Dgraph and returns a map. - """ - def transform_mutation(%ExDgraph.Api.Assigned{context: context, uids: uids}) do - %{ - context: context, - uids: uids - } - end -end diff --git a/test/test_helper.exs b/test/test_helper.exs index 041c4f8..f760f48 100755 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -62,7 +62,7 @@ defmodule ExDgraph.TestHelper do def import_starwars_sample(conn) do ExDgraph.alter(conn, %{schema: @starwars_schema}) - {:ok, _} = ExDgraph.mutation(conn, @starwars_creation_mutation) + {:ok, _, _} = ExDgraph.mutate(conn, @starwars_creation_mutation) end def starwars_creation_mutation() do From 8ada086c37e1d58238a06ff430fefaeb80e11145 Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Tue, 10 Sep 2019 22:27:23 +0200 Subject: [PATCH 60/68] Use proper alias. --- lib/exdgraph/operation.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/exdgraph/operation.ex b/lib/exdgraph/operation.ex index 92322c2..b59829c 100644 --- a/lib/exdgraph/operation.ex +++ b/lib/exdgraph/operation.ex @@ -14,14 +14,14 @@ defmodule ExDgraph.Operation do end defimpl DBConnection.Query, for: ExDgraph.Operation do - alias ExDgraph.{Operation, Payload} + alias ExDgraph.{Api, Operation, Payload} @doc """ This function is called to decode a result after it is returned by a connection callback module. """ def decode( _query, - %ExDgraph.Api.Payload{Data: data} = _result, + %Api.Payload{Data: data} = _result, _opts ) do %Payload{ From ff8244f3686b40bb8ef14bea922d4550a9aacfee Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Tue, 10 Sep 2019 22:53:47 +0200 Subject: [PATCH 61/68] Rename txn_context in result structs. --- lib/exdgraph/mutation.ex | 4 ++-- lib/exdgraph/query.ex | 2 +- lib/exdgraph/structs.ex | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/exdgraph/mutation.ex b/lib/exdgraph/mutation.ex index 1d86e94..f8d65ca 100644 --- a/lib/exdgraph/mutation.ex +++ b/lib/exdgraph/mutation.ex @@ -44,7 +44,7 @@ defimpl DBConnection.Query, for: ExDgraph.Mutation do %MutationResult{ data: data, - context: context, + txn_context: context, latency: latency, uids: uids } @@ -60,7 +60,7 @@ defimpl DBConnection.Query, for: ExDgraph.Mutation do ) when is_nil(set_json) and is_binary(statement) do %MutationResult{ - context: context, + txn_context: context, latency: latency, uids: uids } diff --git a/lib/exdgraph/query.ex b/lib/exdgraph/query.ex index 95ac4e9..1a871f0 100644 --- a/lib/exdgraph/query.ex +++ b/lib/exdgraph/query.ex @@ -35,7 +35,7 @@ defimpl DBConnection.Query, for: ExDgraph.Query do %QueryResult{ data: data, schema: schema, - txn: txn + txn_context: txn } end diff --git a/lib/exdgraph/structs.ex b/lib/exdgraph/structs.ex index a245eaf..ebdb06e 100644 --- a/lib/exdgraph/structs.ex +++ b/lib/exdgraph/structs.ex @@ -25,11 +25,11 @@ defmodule ExDgraph.QueryResult do @type t :: %__MODULE__{ data: %{optional(any) => any}, schema: [any()], - txn: %ExDgraph.Api.TxnContext{}, + txn_context: %ExDgraph.Api.TxnContext{}, uids: map() | nil } - defstruct [:data, :schema, :txn, :uids] + defstruct [:data, :schema, :txn_context, :uids] end defmodule ExDgraph.MutationResult do @@ -42,11 +42,11 @@ defmodule ExDgraph.MutationResult do @type t :: %__MODULE__{ data: %{optional(any) => any}, uids: %{String.t() => String.t()}, - context: Api.TxnContext.t() | nil, + txn_context: Api.TxnContext.t() | nil, latency: Api.Latency.t() | nil } - defstruct [:data, :uids, :context, :latency] + defstruct [:data, :uids, :txn_context, :latency] end defmodule ExDgraph.Payload do From 2711bc7497e758267bdbac8b4b0f6ed13a6ec57b Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Tue, 10 Sep 2019 22:53:52 +0200 Subject: [PATCH 62/68] Cleanup. --- lib/exdgraph.ex | 107 ------------------------------------------------ 1 file changed, 107 deletions(-) diff --git a/lib/exdgraph.ex b/lib/exdgraph.ex index 33a65bd..197838d 100755 --- a/lib/exdgraph.ex +++ b/lib/exdgraph.ex @@ -619,113 +619,6 @@ defmodule ExDgraph do end end - @doc """ - The same as `mutation/2` but raises an `ExDgraph.Exception` if it fails. - Returns the server response otherwise. - """ - # @spec mutation!(conn, String.t()) :: ExDgraph.MutationResult | ExDgraph.Exception - # defdelegate mutation!(conn, statement), to: Mutation - - @doc """ - Allow you to pass a map to insert into the database. The function sends the mutation to the server and returns `{:ok, result}` or `{:error, error}` otherwise. Internally it uses Dgraphs `set_json`. - The `result` is a map of all values you have passed in but with the field `uid` populated from the database. - - ## Examples - - ```elixir - map = %{ - name: "Alice", - friends: %{ - name: "Betty" - } - } - ``` - - iex> ExDgraph.set_map(conn, map) - %{:ok, - %{ - context: %ExDgraph.Api.TxnContext{ - aborted: false, - commit_ts: 1703, - keys: [], - lin_read: %ExDgraph.Api.LinRead{ids: %{1 => 1508}}, - start_ts: 1702 - }, - result: %{ - friends: [%{name: "Betty", uid: "0xd82"}], - name: "Alice", - uid: "0xd81" - }, - uids: %{ - "763d617a-af34-4ff9-9863-e072bf85146d" => "0xd82", - "e94713a5-54a7-4e36-8ab8-0d3019409892" => "0xd81" - } - } - } - """ - # @spec set_map(conn, Map.t()) :: {:ok, ExDgraph.Result} | {:error, ExDgraph.Error} - # defdelegate set_map(conn, map), to: Mutation - - @doc """ - The same as `set_map/2` but raises an `ExDgraph.Exception` if it fails. - Returns the server response otherwise. - """ - # @spec set_map!(conn, Map.t()) :: {:ok, ExDgraph.Result} | {:error, ExDgraph.Error} - # defdelegate set_map!(conn, map), to: Mutation - - @doc """ - This function allow you to convert an struct type into a mutation (json based) type in dgraph. - It also allows you to enforce the schema in database (adding a "class name" info before every predicate, this class name is the single name given to the elixir module struct) by activating the `enforce_struct_schema` in the config files. - The function sends the mutation to the server and returns `{:ok, result}` or `{:error, error}` otherwise. - Internally it uses Dgraphs `set_json`. - The `result` is a map without the "classname" (you can apply the built-in function `struct` to convert the map into the struct you desire) of all values you have passed in but with the field `uid` populated from the database. - - ## Examples - - ```elixir - struct_data = %MutationTest.Person{ - name: "Alice", - identifier: "alice_json", - dogs: [ - %MutationTest.Dog{ - name: "Bello" - } - ] - } - ``` - - iex> ExDgraph.set_struct(conn, struct_data) - %{:ok, - %{ - context: %ExDgraph.Api.TxnContext{ - aborted: false, - commit_ts: 1703, - keys: [], - lin_read: %ExDgraph.Api.LinRead{ids: %{1 => 1508}}, - start_ts: 1702 - }, - result: %{ - dogs: [%{name: "Bello", uid: "0xd82"}], - name: "Alice", - uid: "0xd81" - }, - uids: %{ - "763d617a-af34-4ff9-9863-e072bf85146d" => "0xd82", - "e94713a5-54a7-4e36-8ab8-0d3019409892" => "0xd81" - } - } - } - """ - # @spec set_struct(conn, Map.t()) :: {:ok, ExDgraph.Result} | {:error, ExDgraph.Error} - # defdelegate set_struct(conn, map), to: Mutation - - @doc """ - The same as `set_struct/2` but raises an `ExDgraph.Exception` if it fails. - Returns the server response otherwise. - """ - # @spec set_struct!(conn, Map.t()) :: {:ok, ExDgraph.Result} | {:error, ExDgraph.Error} - # defdelegate set_struct!(conn, map), to: Mutation - ## Operation ###################### From 5d2d1bf4c2236054397c714a86c70db0f6564543 Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Tue, 10 Sep 2019 23:39:02 +0200 Subject: [PATCH 63/68] Implement handle_execute for mutations. --- lib/exdgraph/protocol.ex | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/lib/exdgraph/protocol.ex b/lib/exdgraph/protocol.ex index ef142c0..fc47987 100755 --- a/lib/exdgraph/protocol.ex +++ b/lib/exdgraph/protocol.ex @@ -137,12 +137,18 @@ defmodule ExDgraph.Protocol do %Mutation{statement: statement, set_json: set_json} = query, _params, _, - state + %{channel: channel, opts: opts} = state ) do - dgraph_query = + request = ExDgraph.Api.Mutation.new(set_nquads: statement, set_json: set_json, commit_now: true) - do_mutate(state, dgraph_query, query) + case ExDgraph.Api.Dgraph.Stub.mutate(channel, request, timeout: opts[:timeout]) do + {:ok, res} -> + {:ok, query, res, state} + + {:error, error} -> + {:error, %Error{action: :query, code: error.status, reason: error.message}, state} + end end @impl true @@ -183,19 +189,6 @@ defmodule ExDgraph.Protocol do {:halt, nil, state} end - defp do_mutate(%{channel: channel, opts: opts} = state, dgraph_query, query) do - case ExDgraph.Api.Dgraph.Stub.mutate(channel, dgraph_query, timeout: opts[:timeout]) do - {:ok, res} -> - {:ok, query, res, state} - - {:error, f} -> - raise Exception, code: f.status, message: f.message - end - rescue - e -> - {:error, e, state} - end - @spec default_opts(Keyword.t()) :: Keyword.t() defp default_opts(opts \\ []) do opts From 2cbe97e01539d0c890d68a007bfcc36a17c0c617 Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Tue, 10 Sep 2019 23:39:19 +0200 Subject: [PATCH 64/68] Fix mutation tests. --- lib/exdgraph/mutation.ex | 2 +- test/mutation_test.exs | 263 ++++++++++++++++----------------------- 2 files changed, 111 insertions(+), 154 deletions(-) diff --git a/lib/exdgraph/mutation.ex b/lib/exdgraph/mutation.ex index f8d65ca..ac30f5a 100644 --- a/lib/exdgraph/mutation.ex +++ b/lib/exdgraph/mutation.ex @@ -105,7 +105,7 @@ defimpl DBConnection.Query, for: ExDgraph.Mutation do defp insert_tmp_uids(map, acc \\ 0) defp insert_tmp_uids(map, acc) when is_list(map), - do: Enum.map(map, &insert_tmp_uids(&1, acc).()) + do: Enum.map(map, &insert_tmp_uids(&1, acc)) defp insert_tmp_uids(map, acc) when is_map(map) do map diff --git a/test/mutation_test.exs b/test/mutation_test.exs index 3c5972a..ba4f777 100644 --- a/test/mutation_test.exs +++ b/test/mutation_test.exs @@ -80,166 +80,123 @@ defmodule MutationTest do """ setup do - conn = ExDgraph.conn() - drop_all() - import_starwars_sample() - - on_exit(fn -> - # close channel ? - :ok - end) + {:ok, conn} = ExDgraph.start_link() + ExDgraph.alter(conn, %{drop_all: true}) + import_starwars_sample(conn) [conn: conn] end - test "mutation/2 returns {:ok, mutation_msg} for correct mutation", %{conn: conn} do - {status, mutation_msg} = ExDgraph.mutation(conn, starwars_creation_mutation()) - assert status == :ok - assert mutation_msg.context.aborted == false - end - - test "mutation/2 returns {:error, error} for incorrect mutation", %{conn: conn} do - {status, error} = ExDgraph.mutation(conn, "wrong") - assert status == :error - assert error[:code] == 2 - end - - # TODO: Take care of updates via uid - test "set_map/2 returns {:ok, mutation_msg} for correct mutation", %{conn: conn} do - {status, mutation_msg} = ExDgraph.set_map(conn, @map_insert_mutation) - assert status == :ok - assert mutation_msg.context.aborted == false - query_msg = ExDgraph.Query.query!(conn, @map_insert_check_query) - res = query_msg.result - people = res[:people] - alice = List.first(people) - assert alice[:name] == "Alice" - betty = List.first(alice[:friends]) - assert betty[:name] == "Betty" - end - - test "set_map!/2 returns mutation_message", %{conn: conn} do - mutation_msg = ExDgraph.set_map!(conn, @map_insert_mutation) - assert mutation_msg.context.aborted == false - query_msg = ExDgraph.Query.query!(conn, @map_insert_check_query) - res = query_msg.result - people = res[:people] - alice = List.first(people) - assert alice[:name] == "Alice" - betty = List.first(alice[:friends]) - assert betty[:name] == "Betty" - end - - test "set_map/2 returns result with uids", %{conn: conn} do - {status, mutation_msg} = ExDgraph.set_map(conn, @map_insert_mutation) - assert status == :ok - assert is_map(mutation_msg.result) - mutation_alice = mutation_msg.result - mutation_betty = List.first(mutation_alice[:friends]) - query_msg = ExDgraph.Query.query!(conn, @map_insert_check_query) - query_people = query_msg.result[:people] - query_alice = List.first(query_people) - query_betty = List.first(query_alice[:friends]) - assert mutation_alice[:uid] == query_alice[:uid] - assert mutation_betty[:uid] == query_betty[:uid] - end - - test "set_map!/2 returns result with uids", %{conn: conn} do - mutation_msg = ExDgraph.set_map!(conn, @map_insert_mutation) - assert is_map(mutation_msg.result) - mutation_alice = mutation_msg.result - mutation_betty = List.first(mutation_alice[:friends]) - query_msg = ExDgraph.Query.query!(conn, @map_insert_check_query) - query_people = query_msg.result[:people] - query_alice = List.first(query_people) - query_betty = List.first(query_alice[:friends]) - assert mutation_alice[:uid] == query_alice[:uid] - assert mutation_betty[:uid] == query_betty[:uid] - end - - # TODO: Take care of updates via uid - test "set_map/2 struct returns {:ok, mutation_msg} for correct mutation", %{conn: conn} do - {status, mutation_msg} = ExDgraph.set_struct(conn, @struct_insert_mutation) - assert status == :ok - assert mutation_msg.context.aborted == false - query_msg = ExDgraph.Query.query!(conn, @struct_insert_check_query) - res = query_msg.result - people = res[:people] - alice = List.first(people) - assert alice[:name] == "Alice" - betty = List.first(alice[:dogs]) - assert betty[:name] == "Betty" - some_map = List.first(alice[:some_map]) - assert some_map.some == "value" - bob = List.first(some_map[:map_owner]) - assert bob.name == "Bob" - end - - test "set_map!/2 struct returns mutation_message", %{conn: conn} do - mutation_msg = ExDgraph.set_struct!(conn, @struct_insert_mutation) - assert mutation_msg.context.aborted == false - query_msg = ExDgraph.Query.query!(conn, @struct_insert_check_query) - res = query_msg.result - people = res[:people] - alice = List.first(people) - assert alice[:name] == "Alice" - betty = List.first(alice[:dogs]) - assert betty[:name] == "Betty" - some_map = List.first(alice[:some_map]) - assert some_map.some == "value" - bob = List.first(some_map[:map_owner]) - assert bob.name == "Bob" - end - - test "set_map/2 struct returns result with uids", %{conn: conn} do - {status, mutation_msg} = ExDgraph.set_struct(conn, @struct_insert_mutation) - assert status == :ok - assert is_map(mutation_msg.result) - mutation_alice = mutation_msg.result - mutation_betty = List.first(mutation_alice[:dogs]) - mutation_some_map = mutation_alice[:some_map] - mutation_bob = mutation_some_map[:map_owner] - query_msg = ExDgraph.Query.query!(conn, @struct_insert_check_query) - query_people = query_msg.result[:people] - query_alice = List.first(query_people) - query_betty = List.first(query_alice[:dogs]) - query_some_map = List.first(query_alice[:some_map]) - query_bob = List.first(query_some_map[:map_owner]) - assert mutation_alice[:uid] == query_alice[:uid] - assert mutation_betty[:uid] == query_betty[:uid] - assert mutation_some_map[:uid] == query_some_map[:uid] - assert mutation_bob[:uid] == query_bob[:uid] - end + describe "mutate/3" do + test "it returns {:ok, %Mutation{}, %MutationResult{}} for correct mutation", %{ + conn: conn + } do + {status, _mutation, result} = ExDgraph.mutate(conn, starwars_creation_mutation()) + assert status == :ok + assert result.txn_context.aborted == false + end + + test "it returns {:error, error} for incorrect mutation", %{conn: conn} do + {status, error} = ExDgraph.mutate(conn, "wrong") + assert status == :error + assert error.code == 2 + end + + test "it writes the data to Dgraph", %{ + conn: conn + } do + {status, _mutation, result} = ExDgraph.mutate(conn, @map_insert_mutation) + assert status == :ok + assert result.txn_context.aborted == false + query_msg = ExDgraph.query!(conn, @map_insert_check_query) + res = query_msg.data + people = res[:people] + alice = List.first(people) + assert alice[:name] == "Alice" + betty = List.first(alice[:friends]) + assert betty[:name] == "Betty" + end + + test "it returns result with uids", %{conn: conn} do + {status, _mutation, result} = ExDgraph.mutate(conn, @map_insert_mutation) + assert status == :ok + assert is_map(result.data) + mutation_alice = result.data + mutation_betty = List.first(mutation_alice[:friends]) + query_msg = ExDgraph.query!(conn, @map_insert_check_query) + query_people = query_msg.data[:people] + query_alice = List.first(query_people) + query_betty = List.first(query_alice[:friends]) + assert mutation_alice[:uid] == query_alice[:uid] + assert mutation_betty[:uid] == query_betty[:uid] + end + + test "it updates a node if a uid is present and returns the uid again", %{conn: conn} do + user = %{name: "bob", occupation: "dev"} + {:ok, _mutation, result} = ExDgraph.mutate(conn, user) + + other_mutation = %{ + uid: result.data.uid, + friends: [%{name: "Paul", occupation: "diver"}, %{name: "Lisa", gender: "female"}] + } - test "set_map!/2 struct returns result with uids", %{conn: conn} do - mutation_msg = ExDgraph.set_struct!(conn, @struct_insert_mutation) - assert is_map(mutation_msg.result) - mutation_alice = mutation_msg.result - mutation_betty = List.first(mutation_alice[:dogs]) - mutation_some_map = mutation_alice[:some_map] - mutation_bob = mutation_some_map[:map_owner] - query_msg = ExDgraph.Query.query!(conn, @struct_insert_check_query) - query_people = query_msg.result[:people] - query_alice = List.first(query_people) - query_betty = List.first(query_alice[:dogs]) - query_some_map = List.first(query_alice[:some_map]) - query_bob = List.first(query_some_map[:map_owner]) - assert mutation_alice[:uid] == query_alice[:uid] - assert mutation_betty[:uid] == query_betty[:uid] - assert mutation_some_map[:uid] == query_some_map[:uid] - assert mutation_bob[:uid] == query_bob[:uid] + {:ok, _mutation, result2} = ExDgraph.mutate(conn, other_mutation) + assert result.data.uid == result2.data.uid + end end - test "set_map/2 updates a node if a uid is present and returns the uid again", %{conn: conn} do - user = %{name: "bob", occupation: "dev"} - {:ok, res} = ExDgraph.set_map(conn, user) - - other_mutation = %{ - uid: res.result.uid, - friends: [%{name: "Paul", occupation: "diver"}, %{name: "Lisa", gender: "female"}] - } + describe "mutate!/3" do + test "it returns %MutationResult{} for correct mutation", %{ + conn: conn + } do + result = ExDgraph.mutate!(conn, starwars_creation_mutation()) + assert result.txn_context.aborted == false + end + + test "it raises ExDgraph.Error for incorrect mutation", %{conn: conn} do + assert_raise(ExDgraph.Error, fn -> + ExDgraph.mutate!(conn, "wrong") + end) + end + + test "it writes the data to Dgraph", %{ + conn: conn + } do + result = ExDgraph.mutate!(conn, @map_insert_mutation) + assert result.txn_context.aborted == false + query_msg = ExDgraph.query!(conn, @map_insert_check_query) + res = query_msg.data + people = res[:people] + alice = List.first(people) + assert alice[:name] == "Alice" + betty = List.first(alice[:friends]) + assert betty[:name] == "Betty" + end + + test "it returns result with uids", %{conn: conn} do + result = ExDgraph.mutate!(conn, @map_insert_mutation) + assert is_map(result.data) + mutation_alice = result.data + mutation_betty = List.first(mutation_alice[:friends]) + query_msg = ExDgraph.query!(conn, @map_insert_check_query) + query_people = query_msg.data[:people] + query_alice = List.first(query_people) + query_betty = List.first(query_alice[:friends]) + assert mutation_alice[:uid] == query_alice[:uid] + assert mutation_betty[:uid] == query_betty[:uid] + end + + test "it updates a node if a uid is present and returns the uid again", %{conn: conn} do + user = %{name: "bob", occupation: "dev"} + result = ExDgraph.mutate!(conn, user) + + other_mutation = %{ + uid: result.data.uid, + friends: [%{name: "Paul", occupation: "diver"}, %{name: "Lisa", gender: "female"}] + } - {:ok, res2} = ExDgraph.set_map(conn, other_mutation) - assert res.result.uid == res2.result.uid + result2 = ExDgraph.mutate!(conn, other_mutation) + assert result.data.uid == result2.data.uid + end end end From d67809db405672dd61e1785494ab09cbf44bbe09 Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Wed, 11 Sep 2019 22:02:52 +0200 Subject: [PATCH 65/68] Cleanup. --- lib/exdgraph/mutation.ex | 4 +--- lib/exdgraph/mutation_statement.ex | 16 ---------------- 2 files changed, 1 insertion(+), 19 deletions(-) delete mode 100644 lib/exdgraph/mutation_statement.ex diff --git a/lib/exdgraph/mutation.ex b/lib/exdgraph/mutation.ex index ac30f5a..027a6a9 100644 --- a/lib/exdgraph/mutation.ex +++ b/lib/exdgraph/mutation.ex @@ -3,8 +3,6 @@ defmodule ExDgraph.Mutation do Wrapper for mutations sent to DBConnection. """ - alias ExDgraph.{Exception, Transform} - @type t :: %ExDgraph.Mutation{ statement: String.t() | map() | struct(), set_json: String.t(), @@ -20,7 +18,7 @@ defimpl DBConnection.Query, for: ExDgraph.Mutation do Implementation of `DBConnection.Query` protocol. """ - alias ExDgraph.{Api, Mutation, MutationResult, Transform, Utils} + alias ExDgraph.{Api, Mutation, MutationResult, Utils} @doc """ This function is called to decode a result after it is returned by a connection callback module. diff --git a/lib/exdgraph/mutation_statement.ex b/lib/exdgraph/mutation_statement.ex deleted file mode 100644 index 1ee7235..0000000 --- a/lib/exdgraph/mutation_statement.ex +++ /dev/null @@ -1,16 +0,0 @@ -defmodule ExDgraph.MutationStatement do - @moduledoc false - defstruct statement: "", set_json: "" -end - -defimpl DBConnection.Query, for: ExDgraph.MutationStatement do - alias ExDgraph.Transform - - def describe(query, _), do: query - - def parse(query, _), do: query - - def encode(_query, data, _), do: data - - def decode(_query, result, _opts), do: Transform.transform_mutation(result) -end From 6f86e8d5b18501ecf8446bb6491563d8e76a93fb Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Wed, 11 Sep 2019 22:19:37 +0200 Subject: [PATCH 66/68] Remove elixir_uuid as dependency. --- mix.exs | 1 - mix.lock | 3 --- 2 files changed, 4 deletions(-) diff --git a/mix.exs b/mix.exs index dc3bcdb..fbe9a02 100755 --- a/mix.exs +++ b/mix.exs @@ -38,7 +38,6 @@ defmodule ExDgraph.MixProject do defp deps do [ {:db_connection, "~> 2.1"}, - {:elixir_uuid, "~> 1.2"}, {:grpc, "~> 0.3.1"}, {:jason, "~> 1.1"}, {:protobuf, "~> 0.6.1"}, diff --git a/mix.lock b/mix.lock index 564bd83..733ebd7 100644 --- a/mix.lock +++ b/mix.lock @@ -6,10 +6,7 @@ "cowlib": {:hex, :cowlib, "2.6.0", "8aa629f81a0fc189f261dc98a42243fa842625feea3c7ec56c48f4ccdb55490f", [:rebar3], [], "hexpm"}, "credo": {:hex, :credo, "1.1.4", "c2f3b73c895d81d859cec7fcee7ffdb972c595fd8e85ab6f8c2adbf01cf7c29c", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, "db_connection": {:hex, :db_connection, "2.1.1", "a51e8a2ee54ef2ae6ec41a668c85787ed40cb8944928c191280fe34c15b76ae5", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm"}, - "dialyxir": {:hex, :dialyxir, "1.0.0-rc.6", "78e97d9c0ff1b5521dd68041193891aebebce52fc3b93463c0a6806874557d7d", [:mix], [{:erlex, "~> 0.2.1", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm"}, "earmark": {:hex, :earmark, "1.4.0", "397e750b879df18198afc66505ca87ecf6a96645545585899f6185178433cc09", [:mix], [], "hexpm"}, - "elixir_uuid": {:hex, :elixir_uuid, "1.2.0", "ff26e938f95830b1db152cb6e594d711c10c02c6391236900ddd070a6b01271d", [:mix], [], "hexpm"}, - "erlex": {:hex, :erlex, "0.2.4", "23791959df45fe8f01f388c6f7eb733cc361668cbeedd801bf491c55a029917b", [:mix], [], "hexpm"}, "ex_doc": {:hex, :ex_doc, "0.21.2", "caca5bc28ed7b3bdc0b662f8afe2bee1eedb5c3cf7b322feeeb7c6ebbde089d6", [:mix], [{:earmark, "~> 1.3.3 or ~> 1.4", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm"}, "excoveralls": {:hex, :excoveralls, "0.11.2", "0c6f2c8db7683b0caa9d490fb8125709c54580b4255ffa7ad35f3264b075a643", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, "file_system": {:hex, :file_system, "0.2.7", "e6f7f155970975789f26e77b8b8d8ab084c59844d8ecfaf58cbda31c494d14aa", [:mix], [], "hexpm"}, From 7f87dcbbcdbc653f89a0a89be8e0da510f8b237c Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Wed, 11 Sep 2019 22:34:31 +0200 Subject: [PATCH 67/68] Cleanup Utils. --- lib/exdgraph/expr/uid.ex | 122 --------------------------------------- lib/exdgraph/utils.ex | 85 ++------------------------- test/expr/uid_test.exs | 22 ------- test/utils_test.exs | 94 +++++------------------------- 4 files changed, 20 insertions(+), 303 deletions(-) delete mode 100644 lib/exdgraph/expr/uid.ex delete mode 100644 test/expr/uid_test.exs diff --git a/lib/exdgraph/expr/uid.ex b/lib/exdgraph/expr/uid.ex deleted file mode 100644 index 87fbf5f..0000000 --- a/lib/exdgraph/expr/uid.ex +++ /dev/null @@ -1,122 +0,0 @@ -defmodule ExDgraph.Expr.Uid do - @moduledoc """ - Helper functions to deal with uids. Not in use yet. Intended for query builder. - - https://docs.dgraph.io/query-language/#uid - - ## Syntax Examples: - - q(func: uid()) - predicate @filter(uid(, ..., )) - predicate @filter(uid(a)) for variable a - q(func: uid(a,b)) for variables a and b - - """ - alias ExDgraph.Expr.Uid - alias ExDgraph.Utils - - defstruct [ - :value, - :type - ] - - defmacro __using__(_) do - quote do - def uid(value) do - ExDgraph.Expr.Uid.new(value) - end - end - end - - @types [ - :literal, - :expression - ] - - @doc false - def new(value) when is_binary(value) do - new(value, :literal) - end - - @doc false - def new(value) when is_atom(value) do - new(value, :expression) - end - - @doc false - def new(uids) when is_list(uids) do - # lists of uid literals are rendered inside a `uid()` function (as in @filter) - # lists of uid variables are rendered inside a `uid()` function (as in @filter) - # therefore any list is an uid expression - new(uids, :expression) - end - - @doc false - def new(value, type) - when (is_atom(value) or is_binary(value) or is_list(value)) and type in @types do - %Uid{ - value: value, - type: type - } - end - - @doc false - def as_expression(%Uid{} = u) do - %{u | type: :expression} - end - - @doc false - def as_literal(%Uid{} = u) do - %{u | type: :literal} - end - - @doc false - def as_naked(%Uid{} = u) do - %{u | type: :naked} - end - - @doc false - def render(%Uid{value: value}) when is_atom(value) do - render_expression([value]) - end - - @doc false - def render(%Uid{value: value, type: :literal}) when is_binary(value) do - {:ok, uid_literal} = Utils.as_literal(value, :uid) - uid_literal - end - - @doc false - def render(%Uid{value: value, type: :naked}) when is_binary(value) do - value - end - - @doc false - def render(%Uid{value: value, type: :expression}) when is_atom(value) or is_binary(value) do - render_expression([value]) - end - - @doc false - def render(%Uid{value: value, type: :expression}) when is_list(value) do - render_expression(value) - end - - @doc false - defp render_expression(uids) when is_list(uids) do - args = - uids - |> Enum.map(&to_string/1) - |> Enum.join(", ") - - "uid(" <> args <> ")" - end -end - -defimpl String.Chars, for: ExDgraph.Expr.Uid do - def to_string(uid) do - ExDgraph.Expr.Uid.render(uid) - end -end - -# Source https://github.com/elbow-jason/dgraph_ex -# Copyright (c) 2017 Jason Goldberger diff --git a/lib/exdgraph/utils.ex b/lib/exdgraph/utils.ex index 2385e6b..b40be29 100755 --- a/lib/exdgraph/utils.ex +++ b/lib/exdgraph/utils.ex @@ -1,88 +1,15 @@ defmodule ExDgraph.Utils do @moduledoc "Common utilities" - alias ExDgraph.Expr.Uid + @doc """ + Turns all keys of a nested map into atoms. - def as_rendered(value) do - case value do - x when is_list(x) -> x |> Jason.encode!() - %Date{} = x -> x |> Date.to_iso8601() |> Kernel.<>("T00:00:00.0+00:00") - %DateTime{} = x -> x |> DateTime.to_iso8601() |> String.replace("Z", "+00:00") - x -> x |> to_string - end - end - - def infer_type(type) do - case type do - x when is_boolean(x) -> :bool - x when is_binary(x) -> :string - x when is_integer(x) -> :int - x when is_float(x) -> :float - x when is_list(x) -> :geo - %DateTime{} -> :datetime - %Date{} -> :date - %Uid{} -> :uid - end - end - - def as_literal(value, type) do - case {type, value} do - {:int, v} when is_integer(v) -> {:ok, to_string(v)} - {:float, v} when is_float(v) -> {:ok, as_rendered(v)} - {:bool, v} when is_boolean(v) -> {:ok, as_rendered(v)} - {:string, v} when is_binary(v) -> {:ok, v |> strip_quotes |> wrap_quotes} - {:date, %Date{} = v} -> {:ok, as_rendered(v)} - {:datetime, %DateTime{} = v} -> {:ok, as_rendered(v)} - {:geo, v} when is_list(v) -> check_and_render_geo_numbers(v) - {:uid, v} when is_binary(v) -> {:ok, "<" <> v <> ">"} - _ -> {:error, {:invalidly_typed_value, value, type}} - end - end + ## Example - def as_string(value) do - value - |> as_rendered - |> strip_quotes - |> wrap_quotes - end - - defp check_and_render_geo_numbers(nums) do - if nums |> List.flatten() |> Enum.all?(&is_float/1) do - {:ok, nums |> as_rendered} - else - {:error, :invalid_geo_json} - end - end - - defp wrap_quotes(value) when is_binary(value) do - "\"" <> value <> "\"" - end - - defp strip_quotes(value) when is_binary(value) do - value - |> String.replace(~r/^"/, "") - |> String.replace(~r/"&/, "") - end - - def has_function?(module, func, arity) do - :erlang.function_exported(module, func, arity) - end - - def has_struct?(module) when is_atom(module) do - Code.ensure_loaded?(module) - has_function?(module, :__struct__, 0) - end - - def get_value(params, key, default \\ nil) when is_atom(key) do - str_key = to_string(key) - - cond do - Map.has_key?(params, key) -> Map.get(params, key) - Map.has_key?(params, str_key) -> Map.get(params, str_key) - true -> default - end - end + iex> ExDgraph.Utils.atomify_map_keys(%{"name" => "Ole", "dogs" => [%{"name" => "Pluto"}]}) + %{name: "Ole", dogs: [%{name: "Pluto"}]} + """ def atomify_map_keys(map) when is_map(map) do Enum.reduce(map, %{}, fn {key, value}, acc when is_atom(key) -> diff --git a/test/expr/uid_test.exs b/test/expr/uid_test.exs deleted file mode 100644 index 2821e58..0000000 --- a/test/expr/uid_test.exs +++ /dev/null @@ -1,22 +0,0 @@ -defmodule ExDgraph.Expr.UidTest do - use ExUnit.Case, async: true - - alias ExDgraph.Expr.Uid - use ExDgraph.Expr.Uid - - test "uid given a string renders a plain-old uid literal" do - assert "0x9" |> uid() |> Uid.render() == "<0x9>" - end - - test "uid given an atom renders a uid expression" do - assert :some_atom |> uid() |> Uid.render() == "uid(some_atom)" - end - - test "uid given a list of strings renders a multi-arg uid expr" do - assert ["0x9", "0x10"] |> uid() |> Uid.render() == "uid(0x9, 0x10)" - end - - test "uid given a list of atoms renders a multi-arg uid expr" do - assert [:a, :b, :c] |> uid() |> Uid.render() == "uid(a, b, c)" - end -end diff --git a/test/utils_test.exs b/test/utils_test.exs index 737a84b..89ae27c 100644 --- a/test/utils_test.exs +++ b/test/utils_test.exs @@ -1,89 +1,23 @@ defmodule ExDgraph.Utils.Test do use ExUnit.Case, async: true - doctest ExDgraph.Utils alias ExDgraph.Utils - test "as_rendered/1 float" do - assert Utils.as_rendered(3.14) == "3.14" - end - - test "as_rendered/1 bool" do - assert Utils.as_rendered(true) == "true" - end - - test "as_rendered/1 int" do - assert Utils.as_rendered(1) == "1" - end - - test "as_rendered/1 string" do - assert Utils.as_rendered("beef") == "beef" - end - - test "as_rendered/1 date" do - {:ok, written_on} = Date.new(2017, 8, 5) - assert Utils.as_rendered(written_on) == "2017-08-05T00:00:00.0+00:00" - end + test "atomify_map_keys/1 turns all keys of a nested map into atoms" do + map = %{ + "name" => "Ole", + "dogs" => [%{"name" => "Pluto"}], + "friend" => %{"name" => "Eleasar"}, + already_atom: "value" + } - test "as_rendered/1 datetime" do - {:ok, written_at, 0} = DateTime.from_iso8601("2017-08-05T22:32:36.000+00:00") - assert Utils.as_rendered(written_at) == "2017-08-05T22:32:36.000+00:00" - end - - test "as_rendered/1 geo (json)" do - assert Utils.as_rendered([-111.925278, 33.501324]) == "[-111.925278,33.501324]" - end - - test "as_literal/2 float" do - assert Utils.as_literal(3.14, :float) == {:ok, "3.14"} - end - - test "as_literal/2 bool" do - assert Utils.as_literal(true, :bool) == {:ok, "true"} - end - - test "as_literal/2 int" do - assert Utils.as_literal(1, :int) == {:ok, "1"} - end + result = Utils.atomify_map_keys(map) - test "as_literal/2 string" do - assert Utils.as_literal("beef", :string) == {:ok, "\"beef\""} - end - - test "as_literal/2 date" do - {:ok, written_on} = Date.new(2017, 8, 5) - assert Utils.as_literal(written_on, :date) == {:ok, "2017-08-05T00:00:00.0+00:00"} - end - - test "as_literal/2 datetime" do - {:ok, written_at, 0} = DateTime.from_iso8601("2017-08-05T22:32:36.000+00:00") - assert Utils.as_literal(written_at, :datetime) == {:ok, "2017-08-05T22:32:36.000+00:00"} - end - - test "as_literal/2 geo (json)" do - assert Utils.as_literal([-111.925278, 33.501324], :geo) == {:ok, "[-111.925278,33.501324]"} - end - - test "as_literal/2 type error" do - assert Utils.as_literal("Eef", :int) == {:error, {:invalidly_typed_value, "Eef", :int}} - end - - test "as_literal/2 uid" do - assert Utils.as_literal("beef", :uid) == {:ok, ""} - end - - test "has_struct?/1 returns false for non-struct-modules" do - assert Utils.has_struct?(Path) == false - end - - test "has_struct?/1 returns false for non-struct-atoms" do - assert Utils.has_struct?(:ok) == false - end - - test "has_struct?/1 returns true for struct-having-modules" do - assert Utils.has_struct?(URI) == true + assert result == %{ + name: "Ole", + dogs: [%{name: "Pluto"}], + friend: %{name: "Eleasar"}, + already_atom: "value" + } end end - -# Copyright (c) 2017 Jason Goldberger -# Source https://github.com/elbow-jason/dgraph_ex From bc49c1808681d24b6dd4fba08380e42d4d56edc3 Mon Sep 17 00:00:00 2001 From: Ole Spaarmann Date: Wed, 11 Sep 2019 22:41:07 +0200 Subject: [PATCH 68/68] Add function query_schema/2 and query_schema!/2 --- lib/exdgraph.ex | 86 +++++++++++++++++++++++++++++++++++++++++++++ test/query_test.exs | 17 ++++++++- 2 files changed, 102 insertions(+), 1 deletion(-) diff --git a/lib/exdgraph.ex b/lib/exdgraph.ex index 197838d..018beeb 100755 --- a/lib/exdgraph.ex +++ b/lib/exdgraph.ex @@ -512,6 +512,92 @@ defmodule ExDgraph do end end + @doc """ + Queries the current Dgraph schema and returns `{:ok, result}` or + `{:error, error}` if it fails. + + ## Parameters + + - `conn`: The pool name from `ExDgraph.conn()`. + - `opts`: Options for DBConnection. + + ## Examples + + iex> ExDgraph.query_schema(conn) + %ExDgraph.QueryResult{ + data: %{ + schema: [ + %{list: true, predicate: "_predicate_", type: "string"}, + %{ + count: true, + index: true, + predicate: "name", + tokenizer: ["exact", "term"], + type: "string" + }, + ] + }, + schema: [ + %ExDgraph.Api.SchemaNode{ + count: false, + index: false, + lang: false, + list: true, + predicate: "_predicate_", + reverse: false, + tokenizer: [], + type: "string", + upsert: false + }, + %ExDgraph.Api.SchemaNode{ + count: true, + index: true, + lang: false, + list: false, + predicate: "name", + reverse: false, + tokenizer: ["exact", "term"], + type: "string", + upsert: false + }, + ], + txn_context: %ExDgraph.Api.TxnContext{ + aborted: false, + commit_ts: 0, + keys: [], + lin_read: nil, + preds: [], + start_ts: 130675 + }, + uids: nil + } + + + """ + @spec query!(conn, Keyword.t()) :: {:ok, ExDgraph.QueryResult} | {:ok, ExDgraph.Error} + def query_schema(conn, opts \\ []) do + case query(conn, "schema{}", opts) do + {:ok, _query, result} -> + {:ok, result} + + {:error, error} -> + {:error, error} + end + end + + @doc """ + Same as `query_schema/2 but raises `ExDgraph.Error` if it fails. + """ + def query_schema!(conn, opts \\ []) do + case query_schema(conn, opts) do + {:ok, result} -> + result + + {:error, error} -> + raise error + end + end + ## Mutation ###################### diff --git a/test/query_test.exs b/test/query_test.exs index 56480f8..1f0240c 100644 --- a/test/query_test.exs +++ b/test/query_test.exs @@ -57,9 +57,24 @@ defmodule ExDgraph.QueryTest do assert length(starwars.starring) == 3 end - test "query!/2 raises ExDgraph.Exception", %{conn: conn} do + test "query!/2 raises ExDgraph.Error", %{conn: conn} do assert_raise ExDgraph.Error, fn -> ExDgraph.query!(conn, "wrong") end end + + test "query_schema/2 returns the current schema", %{conn: conn} do + {status, %QueryResult{} = result} = ExDgraph.query_schema(conn) + assert status == :ok + schema = result.schema + data = result.data + + assert Enum.any?(schema, fn %{__struct__: struct, predicate: predicate} -> + struct == ExDgraph.Api.SchemaNode and predicate == "name" + end) + + assert Enum.any?(data.schema, fn %{predicate: predicate} -> + predicate == "name" + end) + end end