From d7af9034c121493234975591575a12800cb5a06c Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Mon, 21 Oct 2019 12:47:22 +0300 Subject: [PATCH 01/12] add ipc client --- .../lib/ethereum_jsonrpc/ipc/IPCClient.ex | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/ipc/IPCClient.ex diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/ipc/IPCClient.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/ipc/IPCClient.ex new file mode 100644 index 0000000000..ecadc47805 --- /dev/null +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/ipc/IPCClient.ex @@ -0,0 +1,58 @@ +defmodule EthereumJSONRPC.HTTP.IPCCLient do + use GenServer + @moduledoc false + + def start_link(state \\ []) do + GenServer.start_link(__MODULE__, Keyword.merge(state, socket: nil)) + end + + def init(state) do + opts = [:binary, active: false, reuseaddr: true] + + response = :gen_tcp.connect({:local, state[:path]}, 0, opts) + + case response do + {:ok, socket} -> {:ok, Keyword.put(state, :socket, socket)} + {:error, reason} -> {:error, reason} + end + end + + def post(pid, request) do + GenServer.call(pid, {:request, request}) + end + + def receive_response(data, socket, timeout, result \\ <<>>) + + def receive_response({:error, reason}, _socket, _timeout, _result) do + {:error, reason} + end + + def receive_response(:ok, socket, timeout, result) do + with {:ok, response} <- :gen_tcp.recv(socket, 0, timeout) do + new_result = result <> response + + if String.ends_with?(response, "\n") do + {:ok, new_result} + else + receive_response(:ok, socket, timeout, new_result) + end + end + end + + def receive_response(data, _socket, _timeout, _result) do + {:error, data} + end + + def handle_call( + {:request, request}, + _from, + [socket: socket, path: _, ipc_request_timeout: timeout] = state + ) do + response = + socket + |> :gen_tcp.send(request) + |> receive_response(socket, timeout) + + {:reply, response, state} + end +end From 87b1d515534391e4286ce4e878446faf7eaeca6a Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Mon, 21 Oct 2019 16:37:29 +0300 Subject: [PATCH 02/12] add json_rpc method --- .../{ipc/IPCClient.ex => ipc.ex} | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) rename apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/{ipc/IPCClient.ex => ipc.ex} (63%) diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/ipc/IPCClient.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/ipc.ex similarity index 63% rename from apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/ipc/IPCClient.ex rename to apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/ipc.ex index ecadc47805..1d2d7dcea3 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/ipc/IPCClient.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/ipc.ex @@ -1,7 +1,9 @@ -defmodule EthereumJSONRPC.HTTP.IPCCLient do +defmodule EthereumJSONRPC.IPC do use GenServer @moduledoc false + # Server + def start_link(state \\ []) do GenServer.start_link(__MODULE__, Keyword.merge(state, socket: nil)) end @@ -55,4 +57,31 @@ defmodule EthereumJSONRPC.HTTP.IPCCLient do {:reply, response, state} end + + # Client + + def json_rpc(pid, payload, _opts) do + with {:ok, response} <- post(pid, payload), + {:ok, decoded_body} <- Jason.decode(response) do + case decoded_body do + %{"error" => error} -> {:error, error} + result = [%{} | _] -> {:ok, format_batch(result)} + result -> {:ok, Map.get(result, "result")} + end + else + {:error, %Jason.DecodeError{data: ""}} -> {:error, :empty_response} + {:error, error} -> {:error, {:invalid_json, error}} + {:error, error} -> {:error, error} + end + end + + defp format_batch(list) do + list + |> Enum.sort(fn %{"id" => id1}, %{"id" => id2} -> + id1 <= id2 + end) + |> Enum.map(fn %{"result" => result} -> + result + end) + end end From f03ead074cac4b19998a9183090ec30e42260cf6 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 22 Oct 2019 14:25:36 +0300 Subject: [PATCH 03/12] start ipc server --- apps/ethereum_jsonrpc/config/config.exs | 5 +++++ .../lib/ethereum_jsonrpc/application.ex | 10 +++++++++- apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/ipc.ex | 15 +++++++++++++-- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/apps/ethereum_jsonrpc/config/config.exs b/apps/ethereum_jsonrpc/config/config.exs index aa5ec08b49..a311073443 100644 --- a/apps/ethereum_jsonrpc/config/config.exs +++ b/apps/ethereum_jsonrpc/config/config.exs @@ -9,6 +9,11 @@ config :ethereum_jsonrpc, EthereumJSONRPC.RequestCoordinator, wait_per_timeout: :timer.seconds(20), max_jitter: :timer.seconds(2) +config :ethereum_jsonrpc, + rpc_transport: if(System.get_env("ETHEREUM_JSONRPC_JSON_RPC_TRANSPORT", "http") == "http", do: :http, else: :ipc), + ipc_path: System.get_env("IPC_PATH") + + # Add this configuration to add global RPC request throttling. # throttle_rate_limit: 250, # throttle_rolling_window_opts: [ diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/application.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/application.ex index 8c902e8f26..38af69a337 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/application.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/application.ex @@ -5,7 +5,7 @@ defmodule EthereumJSONRPC.Application do use Application - alias EthereumJSONRPC.{RequestCoordinator, RollingWindow} + alias EthereumJSONRPC.{IPC, RequestCoordinator, RollingWindow} @impl Application def start(_type, _args) do @@ -17,6 +17,7 @@ defmodule EthereumJSONRPC.Application do :hackney_pool.child_spec(:ethereum_jsonrpc, recv_timeout: 60_000, timeout: 60_000, max_connections: 1000), Supervisor.child_spec({RollingWindow, [rolling_window_opts]}, id: RollingWindow.ErrorThrottle) ] + |> add_ipc_client() |> add_throttle_rolling_window(config) |> Supervisor.start_link(strategy: :one_for_one, name: EthereumJSONRPC.Supervisor) end @@ -37,4 +38,11 @@ defmodule EthereumJSONRPC.Application do children end end + + defp add_ipc_client(children) do + case Application.get_env(:ethereum_jsonrpc, :rpc_transport) do + :ipc -> [IPC.child_spec([path: Application.get_env(:ethereum_jsonrpc, :ipc_path)]) | children] + _ -> children + end + end end diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/ipc.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/ipc.ex index 1d2d7dcea3..6c27ce4fcb 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/ipc.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/ipc.ex @@ -4,8 +4,19 @@ defmodule EthereumJSONRPC.IPC do # Server - def start_link(state \\ []) do - GenServer.start_link(__MODULE__, Keyword.merge(state, socket: nil)) + def child_spec(opts) do + IO.inspect(opts) + default = %{ + id: __MODULE__, + start: {__MODULE__, :start_link, opts}, + type: :worker + } + + Supervisor.child_spec(default, []) + end + + def start_link({:path, path}) do + GenServer.start_link(__MODULE__, [path: path, socket: nil]) end def init(state) do From 141ebda719d190faf711c37fa7d61142d1d8dc24 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 22 Oct 2019 14:53:35 +0300 Subject: [PATCH 04/12] fix message handling --- apps/ethereum_jsonrpc/config/config.exs | 1 - .../lib/ethereum_jsonrpc/application.ex | 4 ++-- apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/ipc.ex | 11 +++++------ apps/indexer/config/dev/parity.exs | 2 +- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/apps/ethereum_jsonrpc/config/config.exs b/apps/ethereum_jsonrpc/config/config.exs index a311073443..f7cf34afc9 100644 --- a/apps/ethereum_jsonrpc/config/config.exs +++ b/apps/ethereum_jsonrpc/config/config.exs @@ -13,7 +13,6 @@ config :ethereum_jsonrpc, rpc_transport: if(System.get_env("ETHEREUM_JSONRPC_JSON_RPC_TRANSPORT", "http") == "http", do: :http, else: :ipc), ipc_path: System.get_env("IPC_PATH") - # Add this configuration to add global RPC request throttling. # throttle_rate_limit: 250, # throttle_rolling_window_opts: [ diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/application.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/application.ex index 38af69a337..394c4de070 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/application.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/application.ex @@ -17,8 +17,8 @@ defmodule EthereumJSONRPC.Application do :hackney_pool.child_spec(:ethereum_jsonrpc, recv_timeout: 60_000, timeout: 60_000, max_connections: 1000), Supervisor.child_spec({RollingWindow, [rolling_window_opts]}, id: RollingWindow.ErrorThrottle) ] - |> add_ipc_client() |> add_throttle_rolling_window(config) + |> add_ipc_client() |> Supervisor.start_link(strategy: :one_for_one, name: EthereumJSONRPC.Supervisor) end @@ -41,7 +41,7 @@ defmodule EthereumJSONRPC.Application do defp add_ipc_client(children) do case Application.get_env(:ethereum_jsonrpc, :rpc_transport) do - :ipc -> [IPC.child_spec([path: Application.get_env(:ethereum_jsonrpc, :ipc_path)]) | children] + :ipc -> [IPC.child_spec(path: Application.get_env(:ethereum_jsonrpc, :ipc_path)) | children] _ -> children end end diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/ipc.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/ipc.ex index 6c27ce4fcb..992b99226b 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/ipc.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/ipc.ex @@ -5,7 +5,6 @@ defmodule EthereumJSONRPC.IPC do # Server def child_spec(opts) do - IO.inspect(opts) default = %{ id: __MODULE__, start: {__MODULE__, :start_link, opts}, @@ -16,7 +15,7 @@ defmodule EthereumJSONRPC.IPC do end def start_link({:path, path}) do - GenServer.start_link(__MODULE__, [path: path, socket: nil]) + GenServer.start_link(__MODULE__, [path: path, socket: nil], name: __MODULE__) end def init(state) do @@ -59,20 +58,20 @@ defmodule EthereumJSONRPC.IPC do def handle_call( {:request, request}, _from, - [socket: socket, path: _, ipc_request_timeout: timeout] = state + [socket: socket, path: _] = state ) do response = socket |> :gen_tcp.send(request) - |> receive_response(socket, timeout) + |> receive_response(socket, 500_000) {:reply, response, state} end # Client - def json_rpc(pid, payload, _opts) do - with {:ok, response} <- post(pid, payload), + def json_rpc(payload, _opts) do + with {:ok, response} <- post(__MODULE__, Jason.encode!(payload)), {:ok, decoded_body} <- Jason.decode(response) do case decoded_body do %{"error" => error} -> {:error, error} diff --git a/apps/indexer/config/dev/parity.exs b/apps/indexer/config/dev/parity.exs index 7a5b31a424..b5ec313191 100644 --- a/apps/indexer/config/dev/parity.exs +++ b/apps/indexer/config/dev/parity.exs @@ -3,7 +3,7 @@ use Mix.Config config :indexer, block_interval: :timer.seconds(5), json_rpc_named_arguments: [ - transport: EthereumJSONRPC.HTTP, + transport: EthereumJSONRPC.IPC, transport_options: [ http: EthereumJSONRPC.HTTP.HTTPoison, url: System.get_env("ETHEREUM_JSONRPC_HTTP_URL") || "http://localhost:8545", From f0f3babd38732c2084fd1659d5f355bb1e9f408a Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 22 Oct 2019 16:02:39 +0300 Subject: [PATCH 05/12] format result from ipc --- .../lib/ethereum_jsonrpc/http.ex | 6 ++-- .../lib/ethereum_jsonrpc/ipc.ex | 29 ++++++++++--------- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/http.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/http.ex index 6d9f946554..8e2c47dd0a 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/http.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/http.ex @@ -132,7 +132,7 @@ defmodule EthereumJSONRPC.HTTP do # restrict response to only those fields supported by the JSON-RPC 2.0 standard, which means that level of keys is # validated, so we can indicate that with switch to atom keys. - defp standardize_response(%{"jsonrpc" => "2.0" = jsonrpc, "id" => id} = unstandardized) do + def standardize_response(%{"jsonrpc" => "2.0" = jsonrpc, "id" => id} = unstandardized) do # Nethermind return string ids id = quantity_to_integer(id) @@ -155,8 +155,8 @@ defmodule EthereumJSONRPC.HTTP do # restrict error to only those fields supported by the JSON-RPC 2.0 standard, which means that level of keys is # validated, so we can indicate that with switch to atom keys. - defp standardize_error(%{"code" => code, "message" => message} = unstandardized) - when is_integer(code) and is_binary(message) do + def standardize_error(%{"code" => code, "message" => message} = unstandardized) + when is_integer(code) and is_binary(message) do standardized = %{code: code, message: message} case Map.fetch(unstandardized, "data") do diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/ipc.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/ipc.ex index 992b99226b..a387f9547f 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/ipc.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/ipc.ex @@ -2,6 +2,8 @@ defmodule EthereumJSONRPC.IPC do use GenServer @moduledoc false + import EthereumJSONRPC.HTTP, only: [standardize_response: 1] + # Server def child_spec(opts) do @@ -74,9 +76,20 @@ defmodule EthereumJSONRPC.IPC do with {:ok, response} <- post(__MODULE__, Jason.encode!(payload)), {:ok, decoded_body} <- Jason.decode(response) do case decoded_body do - %{"error" => error} -> {:error, error} - result = [%{} | _] -> {:ok, format_batch(result)} - result -> {:ok, Map.get(result, "result")} + %{"error" => error} -> + {:error, error} + + result = [%{} | _] -> + list = + result + |> Enum.reverse() + |> List.flatten() + |> Enum.map(&standardize_response/1) + + {:ok, list} + + result -> + {:ok, Map.get(result, "result")} end else {:error, %Jason.DecodeError{data: ""}} -> {:error, :empty_response} @@ -84,14 +97,4 @@ defmodule EthereumJSONRPC.IPC do {:error, error} -> {:error, error} end end - - defp format_batch(list) do - list - |> Enum.sort(fn %{"id" => id1}, %{"id" => id2} -> - id1 <= id2 - end) - |> Enum.map(fn %{"result" => result} -> - result - end) - end end From 2b4fa54a8eb257d68b46a5f1dd4fcf20957be8d5 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 22 Oct 2019 16:09:45 +0300 Subject: [PATCH 06/12] allow to select IPC transport --- apps/indexer/config/dev/ganache.exs | 6 +++++- apps/indexer/config/dev/geth.exs | 6 +++++- apps/indexer/config/dev/parity.exs | 7 ++++++- apps/indexer/config/dev/rsk.exs | 6 +++++- apps/indexer/config/prod/ganache.exs | 6 +++++- apps/indexer/config/prod/geth.exs | 6 +++++- apps/indexer/config/prod/parity.exs | 6 +++++- apps/indexer/config/prod/rsk.exs | 6 +++++- 8 files changed, 41 insertions(+), 8 deletions(-) diff --git a/apps/indexer/config/dev/ganache.exs b/apps/indexer/config/dev/ganache.exs index 337c2317c6..5ffdb90cf7 100644 --- a/apps/indexer/config/dev/ganache.exs +++ b/apps/indexer/config/dev/ganache.exs @@ -3,7 +3,11 @@ use Mix.Config config :indexer, block_interval: :timer.seconds(5), json_rpc_named_arguments: [ - transport: EthereumJSONRPC.HTTP, + transport: + if(System.get_env("ETHEREUM_JSONRPC_JSON_RPC_TRANSPORT", "http") == "http", + do: EthereumJSONRPC.HTTP, + else: EthereumJSONRPC.IPC + ), transport_options: [ http: EthereumJSONRPC.HTTP.HTTPoison, url: System.get_env("ETHEREUM_JSONRPC_HTTP_URL") || "http://localhost:7545", diff --git a/apps/indexer/config/dev/geth.exs b/apps/indexer/config/dev/geth.exs index cf1113119d..a2fb7c8820 100644 --- a/apps/indexer/config/dev/geth.exs +++ b/apps/indexer/config/dev/geth.exs @@ -3,7 +3,11 @@ use Mix.Config config :indexer, block_interval: :timer.seconds(5), json_rpc_named_arguments: [ - transport: EthereumJSONRPC.HTTP, + transport: + if(System.get_env("ETHEREUM_JSONRPC_JSON_RPC_TRANSPORT", "http") == "http", + do: EthereumJSONRPC.HTTP, + else: EthereumJSONRPC.IPC + ), transport_options: [ http: EthereumJSONRPC.HTTP.HTTPoison, url: System.get_env("ETHEREUM_JSONRPC_HTTP_URL") || "https://mainnet.infura.io/8lTvJTKmHPCHazkneJsY", diff --git a/apps/indexer/config/dev/parity.exs b/apps/indexer/config/dev/parity.exs index b5ec313191..4526a76a5b 100644 --- a/apps/indexer/config/dev/parity.exs +++ b/apps/indexer/config/dev/parity.exs @@ -3,7 +3,12 @@ use Mix.Config config :indexer, block_interval: :timer.seconds(5), json_rpc_named_arguments: [ - transport: EthereumJSONRPC.IPC, + transport: + if(System.get_env("ETHEREUM_JSONRPC_JSON_RPC_TRANSPORT", "http") == "http", + do: EthereumJSONRPC.HTTP, + else: EthereumJSONRPC.IPC + ), + else: EthereumJSONRPC.IPC, transport_options: [ http: EthereumJSONRPC.HTTP.HTTPoison, url: System.get_env("ETHEREUM_JSONRPC_HTTP_URL") || "http://localhost:8545", diff --git a/apps/indexer/config/dev/rsk.exs b/apps/indexer/config/dev/rsk.exs index 3f7399e892..8ed8d2badf 100644 --- a/apps/indexer/config/dev/rsk.exs +++ b/apps/indexer/config/dev/rsk.exs @@ -5,7 +5,11 @@ config :indexer, blocks_concurrency: 1, receipts_concurrency: 1, json_rpc_named_arguments: [ - transport: EthereumJSONRPC.HTTP, + transport: + if(System.get_env("ETHEREUM_JSONRPC_JSON_RPC_TRANSPORT", "http") == "http", + do: EthereumJSONRPC.HTTP, + else: EthereumJSONRPC.IPC + ), transport_options: [ http: EthereumJSONRPC.HTTP.HTTPoison, url: System.get_env("ETHEREUM_JSONRPC_HTTP_URL") || "http://localhost:8545", diff --git a/apps/indexer/config/prod/ganache.exs b/apps/indexer/config/prod/ganache.exs index 337c2317c6..5ffdb90cf7 100644 --- a/apps/indexer/config/prod/ganache.exs +++ b/apps/indexer/config/prod/ganache.exs @@ -3,7 +3,11 @@ use Mix.Config config :indexer, block_interval: :timer.seconds(5), json_rpc_named_arguments: [ - transport: EthereumJSONRPC.HTTP, + transport: + if(System.get_env("ETHEREUM_JSONRPC_JSON_RPC_TRANSPORT", "http") == "http", + do: EthereumJSONRPC.HTTP, + else: EthereumJSONRPC.IPC + ), transport_options: [ http: EthereumJSONRPC.HTTP.HTTPoison, url: System.get_env("ETHEREUM_JSONRPC_HTTP_URL") || "http://localhost:7545", diff --git a/apps/indexer/config/prod/geth.exs b/apps/indexer/config/prod/geth.exs index 859d94319c..93af21654b 100644 --- a/apps/indexer/config/prod/geth.exs +++ b/apps/indexer/config/prod/geth.exs @@ -3,7 +3,11 @@ use Mix.Config config :indexer, block_interval: :timer.seconds(5), json_rpc_named_arguments: [ - transport: EthereumJSONRPC.HTTP, + transport: + if(System.get_env("ETHEREUM_JSONRPC_JSON_RPC_TRANSPORT", "http") == "http", + do: EthereumJSONRPC.HTTP, + else: EthereumJSONRPC.IPC + ), transport_options: [ http: EthereumJSONRPC.HTTP.HTTPoison, url: System.get_env("ETHEREUM_JSONRPC_HTTP_URL") || "https://mainnet.infura.io/8lTvJTKmHPCHazkneJsY", diff --git a/apps/indexer/config/prod/parity.exs b/apps/indexer/config/prod/parity.exs index dc9da031d5..38f662598e 100644 --- a/apps/indexer/config/prod/parity.exs +++ b/apps/indexer/config/prod/parity.exs @@ -3,7 +3,11 @@ use Mix.Config config :indexer, block_interval: :timer.seconds(5), json_rpc_named_arguments: [ - transport: EthereumJSONRPC.HTTP, + transport: + if(System.get_env("ETHEREUM_JSONRPC_JSON_RPC_TRANSPORT", "http") == "http", + do: EthereumJSONRPC.HTTP, + else: EthereumJSONRPC.IPC + ), transport_options: [ http: EthereumJSONRPC.HTTP.HTTPoison, url: System.get_env("ETHEREUM_JSONRPC_HTTP_URL"), diff --git a/apps/indexer/config/prod/rsk.exs b/apps/indexer/config/prod/rsk.exs index 81c8c4ac8a..4d90a5b4a3 100644 --- a/apps/indexer/config/prod/rsk.exs +++ b/apps/indexer/config/prod/rsk.exs @@ -5,7 +5,11 @@ config :indexer, blocks_concurrency: 1, receipts_concurrency: 1, json_rpc_named_arguments: [ - transport: EthereumJSONRPC.HTTP, + transport: + if(System.get_env("ETHEREUM_JSONRPC_JSON_RPC_TRANSPORT", "http") == "http", + do: EthereumJSONRPC.HTTP, + else: EthereumJSONRPC.IPC + ), transport_options: [ http: EthereumJSONRPC.HTTP.HTTPoison, url: System.get_env("ETHEREUM_JSONRPC_HTTP_URL"), From 9dbe29f4d3a2f4dd9abc9a7ec02e35091739ad45 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 22 Oct 2019 16:24:22 +0300 Subject: [PATCH 07/12] add CHANGELOG entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e9d619acea..6a36e6c0db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## Current ### Features +- [#2791](https://github.com/poanetwork/blockscout/pull/2791) - add ipc client - [#2772](https://github.com/poanetwork/blockscout/pull/2772) - add token instance images to the token inventory tab - [#2733](https://github.com/poanetwork/blockscout/pull/2733) - Add cache for first page of uncles - [#2735](https://github.com/poanetwork/blockscout/pull/2735) - Add pending transactions cache From 6769f5b31cd251930d3639fe11b08d9e7e60e8e8 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Thu, 24 Oct 2019 12:34:33 +0300 Subject: [PATCH 08/12] use poolboy for concurrent request execution --- .../lib/ethereum_jsonrpc/application.ex | 19 ++++++++++++++-- .../lib/ethereum_jsonrpc/ipc.ex | 22 +++++++------------ apps/ethereum_jsonrpc/mix.exs | 3 ++- 3 files changed, 27 insertions(+), 17 deletions(-) diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/application.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/application.ex index 394c4de070..6c9915f1cd 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/application.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/application.ex @@ -41,8 +41,23 @@ defmodule EthereumJSONRPC.Application do defp add_ipc_client(children) do case Application.get_env(:ethereum_jsonrpc, :rpc_transport) do - :ipc -> [IPC.child_spec(path: Application.get_env(:ethereum_jsonrpc, :ipc_path)) | children] - _ -> children + :ipc -> + [ + :poolboy.child_spec(:worker, poolboy_config(), path: Application.get_env(:ethereum_jsonrpc, :ipc_path)) + | children + ] + + _ -> + children end end + + defp poolboy_config do + [ + {:name, {:local, :ipc_worker}}, + {:worker_module, IPC}, + {:size, 10}, + {:max_overflow, 5} + ] + end end diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/ipc.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/ipc.ex index a387f9547f..7f1ff401b0 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/ipc.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/ipc.ex @@ -6,18 +6,8 @@ defmodule EthereumJSONRPC.IPC do # Server - def child_spec(opts) do - default = %{ - id: __MODULE__, - start: {__MODULE__, :start_link, opts}, - type: :worker - } - - Supervisor.child_spec(default, []) - end - - def start_link({:path, path}) do - GenServer.start_link(__MODULE__, [path: path, socket: nil], name: __MODULE__) + def start_link(opts) do + GenServer.start_link(__MODULE__, Keyword.merge(opts, socket: nil)) end def init(state) do @@ -72,8 +62,8 @@ defmodule EthereumJSONRPC.IPC do # Client - def json_rpc(payload, _opts) do - with {:ok, response} <- post(__MODULE__, Jason.encode!(payload)), + def request(pid, payload) do + with {:ok, response} <- post(pid, Jason.encode!(payload)), {:ok, decoded_body} <- Jason.decode(response) do case decoded_body do %{"error" => error} -> @@ -97,4 +87,8 @@ defmodule EthereumJSONRPC.IPC do {:error, error} -> {:error, error} end end + + def json_rpc(payload, _opts) do + :poolboy.transaction(:ipc_worker, fn pid -> request(pid, payload) end, 600_000) + end end diff --git a/apps/ethereum_jsonrpc/mix.exs b/apps/ethereum_jsonrpc/mix.exs index 240a3aca52..2f5e58dc79 100644 --- a/apps/ethereum_jsonrpc/mix.exs +++ b/apps/ethereum_jsonrpc/mix.exs @@ -91,7 +91,8 @@ defmodule EthereumJsonrpc.MixProject do {:websocket_client, "~> 1.3"}, {:decimal, "~> 1.0"}, {:decorator, "~> 1.2"}, - {:hackney, "~> 1.15.2"} + {:hackney, "~> 1.15.2"}, + {:poolboy, "~> 1.5.2"} ] end end From 247b5fcb9816e675068051cddc45934f3340e1b9 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Thu, 24 Oct 2019 13:09:50 +0300 Subject: [PATCH 09/12] fix address_to_unique_tokens query --- apps/explorer/lib/explorer/chain/token_transfer.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/explorer/lib/explorer/chain/token_transfer.ex b/apps/explorer/lib/explorer/chain/token_transfer.ex index ef0cb4e876..900a57e759 100644 --- a/apps/explorer/lib/explorer/chain/token_transfer.ex +++ b/apps/explorer/lib/explorer/chain/token_transfer.ex @@ -280,7 +280,7 @@ defmodule Explorer.Chain.TokenTransfer do tt in TokenTransfer, left_join: instance in Instance, on: tt.token_contract_address_hash == instance.token_contract_address_hash and tt.token_id == instance.token_id, - where: tt.token_contract_address_hash == ^contract_address_hash and tt.token_id == tt.token_id, + where: tt.token_contract_address_hash == ^contract_address_hash, order_by: [desc: tt.block_number], distinct: tt.token_id, preload: [:to_address], From ea101ffdb603ed0c6be8953bb79a1cc5888e7cd1 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Thu, 24 Oct 2019 13:13:16 +0300 Subject: [PATCH 10/12] add CHANGELOG entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ac75430a7..a777d66dc7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Fixes ### Chore +- [#2801](https://github.com/poanetwork/blockscout/pull/2801) - remove unused clause in address_to_unique_tokens query ## 2.1.0-beta From 87c99e090ca2afb1756b83b8cf5ddd2483787913 Mon Sep 17 00:00:00 2001 From: Andrew Gross Date: Mon, 28 Oct 2019 22:12:02 -0600 Subject: [PATCH 11/12] link docker instructions to doc website --- docker/README.md | 78 ++---------------------------------------------- 1 file changed, 3 insertions(+), 75 deletions(-) diff --git a/docker/README.md b/docker/README.md index ba3458883d..03df272c4a 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,77 +1,5 @@ -# BlockScout Docker integration +# BlockScout Docker Integration -For now this integration is not production ready. It made only for local usage only ! - -## How to use ? -First of all, blockscout requires `PostgreSQL` server for working. -It will be provided by starting script (new docker image will be created named `postgres`) - -**Starting command** -`make start` - will set everything up and start blockscout in container. -To connect it to your local environment you will have to configure it using [env variables](#env-variables) - -Example connecting to local `ganache` instance running on port `2000` on Mac/Windows: -```bash -COIN=DAI \ -ETHEREUM_JSONRPC_VARIANT=ganache \ -ETHEREUM_JSONRPC_HTTP_URL=http://host.docker.internal:2000 \ -ETHEREUM_JSONRPC_WS_URL=ws://host.docker.internal:2000 \ -make start -``` - -Blockscout will be available on `localhost:4000` - -**Note** -On mac/Windows Docker provides with a special URL `host.docker.internal` that will be available into container and routed to your local machine. -On Linux docker is starting using `--network=host` and all services should be available on `localhost` - -### Migrations - -By default, `Makefile` will do migrations for you on `PostgreSQL` creation. -But you could run migrations manually using `make migrate` command. - -**WARNING** Migrations will clean up your local database ! - -## Env variables - -BlockScout supports 3 different JSON RPC Variants. -Variant could be configured using `ETHEREUM_JSONRPC_VARIANT` environment variable. - -Example: -```bash -ETHEREUM_JSONRPC_VARIANT=ganache make start -``` - -Available options are: - - * `parity` - Parity JSON RPC (**Default one**) - * `geth` - Geth JSON RPC - * `ganache` - Ganache JSON RPC - - -| Variable | Description | Default value | -| -------- | ----------- | ------------- | -| `ETHEREUM_JSONRPC_VARIANT` | Variant of your JSON RPC service: `parity`, `geth` or `ganache` | `parity` | -| `ETHEREUM_JSONRPC_HTTP_URL` | HTTP JSON RPC URL Only for `geth` or `ganache` variant | Different per JSONRPC variant | -| `ETHEREUM_JSONRPC_WS_URL` | WS JSON RPC url | Different per JSONRPC variant | -| `ETHEREUM_JSONRPC_TRACE_URL` | Trace URL **Only for `parity` variant** | `http://localhost:8545` | -| `COIN` | Default Coin | `POA` | -| `LOGO` | Coin logo | Empty | -| `NETWORK` | Network | Empty | -| `SUBNETWORK` | Subnetwork | Empty | -| `NETWORK_ICON` | Network icon | Empty | -| `NETWORK_PATH` | Network path | `/` | - - -`ETHEREUM_JSONRPC_HTTP_URL` default values: - - * For `parity` - `http://localhost:8545` - * For `geth` - `https://mainnet.infura.io/8lTvJTKmHPCHazkneJsY` - * For `ganache` - `http://localhost:7545` - -`ETHEREUM_JSONRPC_WS_URL` default values: - - * For `parity` - `ws://localhost:8546` - * For `geth` - `wss://mainnet.infura.io/8lTvJTKmHPCHazkneJsY/ws` - * For `ganache` - `ws://localhost:7545` +This integration is not production ready, and should be used for local BlockScout deployment only. +For usage instructions and ENV variables, see the [docker integration documentation](https://docs.blockscout.com/for-developers/information-and-settings/docker-integration-local-use-only). \ No newline at end of file From fd1f7d8dfdc4fe237d620ded72de26d5ce6b65ae Mon Sep 17 00:00:00 2001 From: Andrew Gross Date: Mon, 28 Oct 2019 22:16:13 -0600 Subject: [PATCH 12/12] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e75b77e04..18b1ad3994 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ ### Fixes ### Chore +- [#2817](https://github.com/poanetwork/blockscout/pull/2817) - move docker integration documentation to blockscout docs ## 2.1.0-beta