diff --git a/CHANGELOG.md b/CHANGELOG.md index 412c19f8ef..b4f7d89a28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ### Features - [#1739](https://github.com/poanetwork/blockscout/pull/1739) - highlight decompiled source code +- [#1742](https://github.com/poanetwork/blockscout/pull/1742) - Support RSK ### Fixes @@ -13,7 +14,9 @@ ### Chore + - [#1749](https://github.com/poanetwork/blockscout/pull/1749) - Replace the link in the footer with the official POA announcements tg channel link - [#1718](https://github.com/poanetwork/blockscout/pull/1718) - Flatten indexer module hierarchy and supervisor tree + - [#1753](https://github.com/poanetwork/blockscout/pull/1753) - Add a check mark to decompiled contract tab ## 1.3.9-beta diff --git a/README.md b/README.md index bb4e3a249b..e323e724dc 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,7 @@ The [development stack page](https://github.com/poanetwork/blockscout/wiki/Devel `cd apps/explorer && npm install; cd -` 7. Update your JSON RPC Variant in `apps/explorer/config/dev.exs` and `apps/indexer/config/dev.exs`. - For `variant`, enter `ganache`, `geth`, or `parity` + For `variant`, enter `ganache`, `geth`, `parity`, or `rsk` 8. Update your JSON RPC Endpoint in `apps/explorer/config/dev/` and `apps/indexer/config/dev/` For the `variant` chosen in step 7, enter the correct information for the corresponding JSON RPC Endpoint in `parity.exs`, `geth.exs`, or `ganache.exs` diff --git a/apps/block_scout_web/config/config.exs b/apps/block_scout_web/config/config.exs index 179e83c8c4..7c341f6f74 100644 --- a/apps/block_scout_web/config/config.exs +++ b/apps/block_scout_web/config/config.exs @@ -93,7 +93,7 @@ config :block_scout_web, BlockScoutWeb.Gettext, locales: ~w(en), default_locale: config :block_scout_web, BlockScoutWeb.SocialMedia, twitter: "PoaNetwork", - telegram: "oraclesnetwork", + telegram: "poa_network", facebook: "PoaNetwork", instagram: "PoaNetwork" diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/api/rpc/rpc_translator.ex b/apps/block_scout_web/lib/block_scout_web/controllers/api/rpc/rpc_translator.ex index b1db31a8f3..a5a4959e8d 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/api/rpc/rpc_translator.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/api/rpc/rpc_translator.ex @@ -13,6 +13,8 @@ defmodule BlockScoutWeb.API.RPC.RPCTranslator do """ + require Logger + import Plug.Conn import Phoenix.Controller, only: [put_view: 2] @@ -28,12 +30,28 @@ defmodule BlockScoutWeb.API.RPC.RPCTranslator do {:ok, conn} <- call_controller(conn, controller, action) do conn else - _ -> + {:error, :no_action} -> conn |> put_status(400) |> put_view(RPCView) |> Controller.render(:error, error: "Unknown action") |> halt() + + {:error, error} -> + Logger.error(fn -> ["Error while calling RPC action", inspect(error)] end) + + conn + |> put_status(500) + |> put_view(RPCView) + |> Controller.render(:error, error: "Something went wrong.") + |> halt() + + _ -> + conn + |> put_status(500) + |> put_view(RPCView) + |> Controller.render(:error, error: "Something went wrong.") + |> halt() end end @@ -46,27 +64,35 @@ defmodule BlockScoutWeb.API.RPC.RPCTranslator do end @doc false - @spec translate_module(map(), String.t()) :: {:ok, module()} | :error - def translate_module(translations, module) do + @spec translate_module(map(), String.t()) :: {:ok, module()} | {:error, :no_action} + defp translate_module(translations, module) do module_lowercase = String.downcase(module) - Map.fetch(translations, module_lowercase) + + case Map.fetch(translations, module_lowercase) do + {:ok, module} -> {:ok, module} + _ -> {:error, :no_action} + end end @doc false - @spec translate_action(String.t()) :: {:ok, atom()} | :error - def translate_action(action) do + @spec translate_action(String.t()) :: {:ok, atom()} | {:error, :no_action} + defp translate_action(action) do action_lowercase = String.downcase(action) {:ok, String.to_existing_atom(action_lowercase)} rescue - ArgumentError -> :error + ArgumentError -> {:error, :no_action} end @doc false - @spec call_controller(Conn.t(), module(), atom()) :: {:ok, Conn.t()} | :error - def call_controller(conn, controller, action) do - {:ok, controller.call(conn, action)} + @spec call_controller(Conn.t(), module(), atom()) :: {:ok, Conn.t()} | {:error, :no_action} | {:error, Exception.t()} + defp call_controller(conn, controller, action) do + if :erlang.function_exported(controller, action, 2) do + {:ok, controller.call(conn, action)} + else + {:error, :no_action} + end rescue - Conn.WrapperError -> - :error + e -> + {:error, Exception.format(:error, e, __STACKTRACE__)} end end diff --git a/apps/block_scout_web/lib/block_scout_web/templates/address/_tabs.html.eex b/apps/block_scout_web/lib/block_scout_web/templates/address/_tabs.html.eex index f981f75ace..8c4a932b76 100644 --- a/apps/block_scout_web/lib/block_scout_web/templates/address/_tabs.html.eex +++ b/apps/block_scout_web/lib/block_scout_web/templates/address/_tabs.html.eex @@ -64,8 +64,9 @@ <%= link( to: address_decompiled_contract_path(@conn, :index, @address.hash), class: "nav-link #{tab_status("decompiled_contracts", @conn.request_path)}") do %> - <%= gettext("Decompiled code") %> - <% end %> + <%= gettext("Decompiled code") %> + + <% end %> <% end %> @@ -131,6 +132,7 @@ to: address_decompiled_contract_path(@conn, :index, @address.hash), class: "dropdown-item #{tab_status("contracts", @conn.request_path)}") do %> <%= gettext("Decompiled code") %> + <% end %> <% end %> <% end %> diff --git a/apps/block_scout_web/lib/block_scout_web/templates/address_decompiled_contract/index.html.eex b/apps/block_scout_web/lib/block_scout_web/templates/address_decompiled_contract/index.html.eex index 64f364d9b2..59c74ed237 100644 --- a/apps/block_scout_web/lib/block_scout_web/templates/address_decompiled_contract/index.html.eex +++ b/apps/block_scout_web/lib/block_scout_web/templates/address_decompiled_contract/index.html.eex @@ -10,7 +10,7 @@

<%= gettext "Decompiler version" %>

-
<%= contract.decompiler_version %>
+
<%= contract.decompiler_version %>

diff --git a/apps/block_scout_web/lib/block_scout_web/templates/layout/_footer.html.eex b/apps/block_scout_web/lib/block_scout_web/templates/layout/_footer.html.eex index 4a15dc8f1e..46a2531b09 100644 --- a/apps/block_scout_web/lib/block_scout_web/templates/layout/_footer.html.eex +++ b/apps/block_scout_web/lib/block_scout_web/templates/layout/_footer.html.eex @@ -17,7 +17,7 @@ "> - "> + ">
diff --git a/apps/block_scout_web/priv/gettext/default.pot b/apps/block_scout_web/priv/gettext/default.pot index 28d3db09d8..6f30fdf2e3 100644 --- a/apps/block_scout_web/priv/gettext/default.pot +++ b/apps/block_scout_web/priv/gettext/default.pot @@ -188,7 +188,7 @@ msgstr "" #, elixir-format #: lib/block_scout_web/templates/address/_tabs.html.eex:40 -#: lib/block_scout_web/templates/address/_tabs.html.eex:113 +#: lib/block_scout_web/templates/address/_tabs.html.eex:114 #: lib/block_scout_web/templates/address/overview.html.eex:59 #: lib/block_scout_web/templates/address_validation/index.html.eex:30 #: lib/block_scout_web/templates/address_validation/index.html.eex:57 @@ -218,7 +218,7 @@ msgstr "" #, elixir-format #: lib/block_scout_web/templates/address/_tabs.html.eex:53 -#: lib/block_scout_web/templates/address/_tabs.html.eex:123 +#: lib/block_scout_web/templates/address/_tabs.html.eex:124 #: lib/block_scout_web/templates/address_validation/index.html.eex:39 #: lib/block_scout_web/templates/api_docs/_action_tile.html.eex:119 #: lib/block_scout_web/templates/api_docs/_action_tile.html.eex:141 @@ -488,7 +488,7 @@ msgstr "" #, elixir-format #: lib/block_scout_web/templates/address/_tabs.html.eex:21 -#: lib/block_scout_web/templates/address/_tabs.html.eex:100 +#: lib/block_scout_web/templates/address/_tabs.html.eex:101 #: lib/block_scout_web/templates/address_internal_transaction/index.html.eex:58 #: lib/block_scout_web/templates/address_validation/index.html.eex:24 #: lib/block_scout_web/templates/transaction/_tabs.html.eex:14 @@ -698,8 +698,8 @@ msgid "Query" msgstr "" #, elixir-format -#: lib/block_scout_web/templates/address/_tabs.html.eex:75 -#: lib/block_scout_web/templates/address/_tabs.html.eex:140 +#: lib/block_scout_web/templates/address/_tabs.html.eex:76 +#: lib/block_scout_web/templates/address/_tabs.html.eex:142 #: lib/block_scout_web/templates/tokens/overview/_tabs.html.eex:33 #: lib/block_scout_web/templates/tokens/overview/_tabs.html.eex:75 #: lib/block_scout_web/views/address_view.ex:298 @@ -891,7 +891,7 @@ msgstr "" #, elixir-format #: lib/block_scout_web/templates/address/_tabs.html.eex:13 -#: lib/block_scout_web/templates/address/_tabs.html.eex:95 +#: lib/block_scout_web/templates/address/_tabs.html.eex:96 #: lib/block_scout_web/templates/address_token/index.html.eex:11 #: lib/block_scout_web/templates/address_token_transfer/index.html.eex:12 #: lib/block_scout_web/templates/address_validation/index.html.eex:11 @@ -948,7 +948,7 @@ msgstr "" #, elixir-format #: lib/block_scout_web/templates/address/_tabs.html.eex:5 -#: lib/block_scout_web/templates/address/_tabs.html.eex:90 +#: lib/block_scout_web/templates/address/_tabs.html.eex:91 #: lib/block_scout_web/templates/address_transaction/index.html.eex:53 #: lib/block_scout_web/templates/address_validation/index.html.eex:14 #: lib/block_scout_web/templates/block_transaction/index.html.eex:13 @@ -1399,7 +1399,7 @@ msgstr "" #, elixir-format #: lib/block_scout_web/templates/address/_tabs.html.eex:30 -#: lib/block_scout_web/templates/address/_tabs.html.eex:106 +#: lib/block_scout_web/templates/address/_tabs.html.eex:107 #: lib/block_scout_web/views/address_view.ex:299 msgid "Coin Balance History" msgstr "" @@ -1714,7 +1714,7 @@ msgstr "" #, elixir-format #: lib/block_scout_web/templates/address/_tabs.html.eex:67 -#: lib/block_scout_web/templates/address/_tabs.html.eex:133 +#: lib/block_scout_web/templates/address/_tabs.html.eex:134 msgid "Decompiled code" msgstr "" diff --git a/apps/block_scout_web/priv/gettext/en/LC_MESSAGES/default.po b/apps/block_scout_web/priv/gettext/en/LC_MESSAGES/default.po index 6dbdb2d676..a1fb23d655 100644 --- a/apps/block_scout_web/priv/gettext/en/LC_MESSAGES/default.po +++ b/apps/block_scout_web/priv/gettext/en/LC_MESSAGES/default.po @@ -188,7 +188,7 @@ msgstr "" #, elixir-format #: lib/block_scout_web/templates/address/_tabs.html.eex:40 -#: lib/block_scout_web/templates/address/_tabs.html.eex:113 +#: lib/block_scout_web/templates/address/_tabs.html.eex:114 #: lib/block_scout_web/templates/address/overview.html.eex:59 #: lib/block_scout_web/templates/address_validation/index.html.eex:30 #: lib/block_scout_web/templates/address_validation/index.html.eex:57 @@ -218,7 +218,7 @@ msgstr "" #, elixir-format #: lib/block_scout_web/templates/address/_tabs.html.eex:53 -#: lib/block_scout_web/templates/address/_tabs.html.eex:123 +#: lib/block_scout_web/templates/address/_tabs.html.eex:124 #: lib/block_scout_web/templates/address_validation/index.html.eex:39 #: lib/block_scout_web/templates/api_docs/_action_tile.html.eex:119 #: lib/block_scout_web/templates/api_docs/_action_tile.html.eex:141 @@ -488,7 +488,7 @@ msgstr "" #, elixir-format #: lib/block_scout_web/templates/address/_tabs.html.eex:21 -#: lib/block_scout_web/templates/address/_tabs.html.eex:100 +#: lib/block_scout_web/templates/address/_tabs.html.eex:101 #: lib/block_scout_web/templates/address_internal_transaction/index.html.eex:58 #: lib/block_scout_web/templates/address_validation/index.html.eex:24 #: lib/block_scout_web/templates/transaction/_tabs.html.eex:14 @@ -698,8 +698,8 @@ msgid "Query" msgstr "" #, elixir-format -#: lib/block_scout_web/templates/address/_tabs.html.eex:75 -#: lib/block_scout_web/templates/address/_tabs.html.eex:140 +#: lib/block_scout_web/templates/address/_tabs.html.eex:76 +#: lib/block_scout_web/templates/address/_tabs.html.eex:142 #: lib/block_scout_web/templates/tokens/overview/_tabs.html.eex:33 #: lib/block_scout_web/templates/tokens/overview/_tabs.html.eex:75 #: lib/block_scout_web/views/address_view.ex:298 @@ -891,7 +891,7 @@ msgstr "" #, elixir-format #: lib/block_scout_web/templates/address/_tabs.html.eex:13 -#: lib/block_scout_web/templates/address/_tabs.html.eex:95 +#: lib/block_scout_web/templates/address/_tabs.html.eex:96 #: lib/block_scout_web/templates/address_token/index.html.eex:11 #: lib/block_scout_web/templates/address_token_transfer/index.html.eex:12 #: lib/block_scout_web/templates/address_validation/index.html.eex:11 @@ -948,7 +948,7 @@ msgstr "" #, elixir-format #: lib/block_scout_web/templates/address/_tabs.html.eex:5 -#: lib/block_scout_web/templates/address/_tabs.html.eex:90 +#: lib/block_scout_web/templates/address/_tabs.html.eex:91 #: lib/block_scout_web/templates/address_transaction/index.html.eex:53 #: lib/block_scout_web/templates/address_validation/index.html.eex:14 #: lib/block_scout_web/templates/block_transaction/index.html.eex:13 @@ -1399,7 +1399,7 @@ msgstr "" #, elixir-format #: lib/block_scout_web/templates/address/_tabs.html.eex:30 -#: lib/block_scout_web/templates/address/_tabs.html.eex:106 +#: lib/block_scout_web/templates/address/_tabs.html.eex:107 #: lib/block_scout_web/views/address_view.ex:299 msgid "Coin Balance History" msgstr "" @@ -1714,7 +1714,7 @@ msgstr "" #, elixir-format #: lib/block_scout_web/templates/address/_tabs.html.eex:67 -#: lib/block_scout_web/templates/address/_tabs.html.eex:133 +#: lib/block_scout_web/templates/address/_tabs.html.eex:134 msgid "Decompiled code" msgstr "" @@ -1723,12 +1723,12 @@ msgstr "" msgid "Decompiled contract code" msgstr "" -#, elixir-format, fuzzy +#, elixir-format #: lib/block_scout_web/templates/address_decompiled_contract/index.html.eex:11 msgid "Decompiler version" msgstr "" -#, elixir-format, fuzzy +#, elixir-format #: lib/block_scout_web/templates/address_contract_verification/new.html.eex:52 msgid "Optimization runs" msgstr "" diff --git a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/rpc_translator_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/rpc_translator_test.exs index 6a847bbc5b..3300085679 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/rpc_translator_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/rpc_translator_test.exs @@ -75,20 +75,4 @@ defmodule BlockScoutWeb.API.RPC.RPCTranslatorTest do assert json_response(result, 200) == %{} end end - - test "translate_module/2" do - assert RPCTranslator.translate_module(%{"test" => __MODULE__}, "tesT") == {:ok, __MODULE__} - assert RPCTranslator.translate_module(%{}, "test") == :error - end - - test "translate_action/1" do - expected = :test_atom - assert RPCTranslator.translate_action("test_atoM") == {:ok, expected} - assert RPCTranslator.translate_action("some_atom_that_should_not_exist") == :error - end - - test "call_controller/3", %{conn: conn} do - assert RPCTranslator.call_controller(conn, TestController, :bad_action) == :error - assert {:ok, %Plug.Conn{}} = RPCTranslator.call_controller(conn, TestController, :test_action) - end end diff --git a/apps/ethereum_jsonrpc/config/config.exs b/apps/ethereum_jsonrpc/config/config.exs index bc82001113..aa5ec08b49 100644 --- a/apps/ethereum_jsonrpc/config/config.exs +++ b/apps/ethereum_jsonrpc/config/config.exs @@ -9,6 +9,14 @@ config :ethereum_jsonrpc, EthereumJSONRPC.RequestCoordinator, wait_per_timeout: :timer.seconds(20), max_jitter: :timer.seconds(2) +# Add this configuration to add global RPC request throttling. +# throttle_rate_limit: 250, +# throttle_rolling_window_opts: [ +# window_count: 4, +# duration: :timer.seconds(15), +# table: EthereumJSONRPC.RequestCoordinator.ThrottleCounter +# ] + config :ethereum_jsonrpc, EthereumJSONRPC.Tracer, service: :ethereum_jsonrpc, adapter: SpandexDatadog.Adapter, diff --git a/apps/ethereum_jsonrpc/config/test.exs b/apps/ethereum_jsonrpc/config/test.exs index e76eab248f..88e4eba302 100644 --- a/apps/ethereum_jsonrpc/config/test.exs +++ b/apps/ethereum_jsonrpc/config/test.exs @@ -7,7 +7,14 @@ config :ethereum_jsonrpc, EthereumJSONRPC.RequestCoordinator, table: EthereumJSONRPC.RequestCoordinator.TimeoutCounter ], wait_per_timeout: 2, - max_jitter: 1 + max_jitter: 1, + # This should not actually limit anything in tests, but it is here to enable the relevant code for testing + throttle_rate_limit: 10_000, + throttle_rolling_window_opts: [ + window_count: 4, + duration: :timer.seconds(1), + table: EthereumJSONRPC.RequestCoordinator.ThrottleCounter + ] config :ethereum_jsonrpc, EthereumJSONRPC.Tracer, disabled?: false diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc.ex index 2e710488bd..33d3abd31b 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc.ex @@ -326,11 +326,18 @@ defmodule EthereumJSONRPC do @doc """ Converts `t:quantity/0` to `t:non_neg_integer/0`. """ - @spec quantity_to_integer(quantity) :: non_neg_integer() + @spec quantity_to_integer(quantity) :: non_neg_integer() | :error def quantity_to_integer("0x" <> hexadecimal_digits) do String.to_integer(hexadecimal_digits, 16) end + def quantity_to_integer(string) do + case Integer.parse(string) do + {integer, ""} -> integer + _ -> :error + end + end + @doc """ Converts `t:non_neg_integer/0` to `t:quantity/0` """ @@ -398,9 +405,13 @@ defmodule EthereumJSONRPC do Converts `t:timestamp/0` to `t:DateTime.t/0` """ def timestamp_to_datetime(timestamp) do - timestamp - |> quantity_to_integer() - |> Timex.from_unix() + case quantity_to_integer(timestamp) do + :error -> + nil + + quantity -> + Timex.from_unix(quantity) + end end defp fetch_blocks_by_params(params, request, json_rpc_named_arguments) diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/application.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/application.ex index e39e9b1d5b..8c902e8f26 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/application.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/application.ex @@ -9,16 +9,32 @@ defmodule EthereumJSONRPC.Application do @impl Application def start(_type, _args) do - rolling_window_opts = - :ethereum_jsonrpc - |> Application.fetch_env!(RequestCoordinator) - |> Keyword.fetch!(:rolling_window_opts) + config = Application.fetch_env!(:ethereum_jsonrpc, RequestCoordinator) - children = [ + rolling_window_opts = Keyword.fetch!(config, :rolling_window_opts) + + [ :hackney_pool.child_spec(:ethereum_jsonrpc, recv_timeout: 60_000, timeout: 60_000, max_connections: 1000), - {RollingWindow, [rolling_window_opts]} + Supervisor.child_spec({RollingWindow, [rolling_window_opts]}, id: RollingWindow.ErrorThrottle) ] + |> add_throttle_rolling_window(config) + |> Supervisor.start_link(strategy: :one_for_one, name: EthereumJSONRPC.Supervisor) + end + + defp add_throttle_rolling_window(children, config) do + if config[:throttle_rate_limit] do + case Keyword.fetch(config, :throttle_rolling_window_opts) do + {:ok, throttle_rolling_window_opts} -> + child = + Supervisor.child_spec({RollingWindow, [throttle_rolling_window_opts]}, id: RollingWindow.ThrottleRateLimit) + + [child | children] - Supervisor.start_link(children, strategy: :one_for_one, name: EthereumJSONRPC.Supervisor) + :error -> + raise "If you have configured `:throttle_rate_limit` you must also configure `:throttle_rolling_window_opts`" + end + else + children + end end end diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/block.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/block.ex index 06ebe479df..3aa6eed029 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/block.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/block.ex @@ -431,7 +431,8 @@ defmodule EthereumJSONRPC.Block do Enum.into(block, %{}, &entry_to_elixir/1) end - defp entry_to_elixir({key, quantity}) when key in ~w(difficulty gasLimit gasUsed number size totalDifficulty) do + defp entry_to_elixir({key, quantity}) + when key in ~w(difficulty gasLimit gasUsed minimumGasPrice number size totalDifficulty) do {key, quantity_to_integer(quantity)} end diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/receipt.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/receipt.ex index 0a20f4ba5d..7bc96509b4 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/receipt.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/receipt.ex @@ -274,10 +274,10 @@ defmodule EthereumJSONRPC.Receipt do defp entry_to_elixir({"status" = key, status}) do case status do - "0x0" -> + zero when zero in ["0x0", "0x00"] -> {:ok, {key, :error}} - "0x1" -> + one when one in ["0x1", "0x01"] -> {:ok, {key, :ok}} # pre-Byzantium / Ethereum Classic on Parity diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/request_coordinator.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/request_coordinator.ex index c72953555b..bd491f25fd 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/request_coordinator.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/request_coordinator.ex @@ -18,9 +18,14 @@ defmodule EthereumJSONRPC.RequestCoordinator do the tracked window * `:max_jitter` - Maximimum amount of time in milliseconds to be added to each wait before multiplied by timeout count + * `:throttle_rolling_window_opts` - Options for the process tracking all requests + * `:window_count` - Number of windows + * `:duration` - Total amount of time to coumt events in milliseconds + * `:table` - name of the ets table to store the data in + * `:throttle_rate_limit` - The total number of requests allowed in the all windows. See the docs for `EthereumJSONRPC.RollingWindow` for more documentation for - `:rolling_window_opts`. + `:rolling_window_opts` and `:throttle_rolling_window_opts`. This is how the wait time for each request is calculated: @@ -33,13 +38,20 @@ defmodule EthereumJSONRPC.RequestCoordinator do config :ethereum_jsonrpc, EthereumJSONRPC.RequestCoordinator, rolling_window_opts: [ window_count: 6, - duration: :timer.seconds(10), + duration: :timer.minutes(1), table: EthereumJSONRPC.RequestCoordinator.TimeoutCounter ], wait_per_timeout: :timer.seconds(10), max_jitter: :timer.seconds(1) + throttle_rate_limit: 60, + throttle_rolling_window_opts: [ + window_count: 3, + duration: :timer.seconds(10), + table: EthereumJSONRPC.RequestCoordinator.RequestCounter + ] With this configuration, timeouts are tracked for 6 windows of 10 seconds for a total of 1 minute. + Requests are tracked for 3 windows of 10 seconds, for a total of 30 seconds, and """ require EthereumJSONRPC.Tracer @@ -47,6 +59,7 @@ defmodule EthereumJSONRPC.RequestCoordinator do alias EthereumJSONRPC.{RollingWindow, Tracer, Transport} @error_key :throttleable_error_count + @throttle_key :throttle_requests_count @doc """ Performs a JSON RPC request and adds necessary backoff. @@ -64,12 +77,19 @@ defmodule EthereumJSONRPC.RequestCoordinator do if sleep_time <= throttle_timeout do :timer.sleep(sleep_time) - - trace_request(request, fn -> - request - |> transport.json_rpc(transport_options) - |> handle_transport_response() - end) + remaining_wait_time = throttle_timeout - sleep_time + + case throttle_request(remaining_wait_time) do + :ok -> + trace_request(request, fn -> + request + |> transport.json_rpc(transport_options) + |> handle_transport_response() + end) + + :error -> + {:error, :timeout} + end else :timer.sleep(throttle_timeout) @@ -91,33 +111,78 @@ defmodule EthereumJSONRPC.RequestCoordinator do defp handle_transport_response({:error, {:bad_gateway, _}} = error) do RollingWindow.inc(table(), @error_key) + inc_throttle_table() error end defp handle_transport_response({:error, :timeout} = error) do RollingWindow.inc(table(), @error_key) + inc_throttle_table() error end - defp handle_transport_response(response), do: response + defp handle_transport_response(response) do + inc_throttle_table() + response + end + + defp inc_throttle_table do + if config(:throttle_rolling_window_opts) do + RollingWindow.inc(throttle_table(), @throttle_key) + end + end + + defp throttle_request( + remaining_time, + rate_limit \\ config(:throttle_rate_limit), + opts \\ config(:throttle_rolling_window_opts) + ) do + if opts[:throttle_rate_limit] && RollingWindow.count(throttle_table(), @throttle_key) >= rate_limit do + if opts[:duration] >= remaining_time do + :timer.sleep(remaining_time) + + :error + else + new_remaining_time = remaining_time - opts[:duration] + :timer.sleep(opts[:duration]) + + throttle_request(new_remaining_time, rate_limit, opts) + end + else + :ok + end + end defp sleep_time do wait_coefficient = RollingWindow.count(table(), @error_key) - jitter = :rand.uniform(config(:max_jitter)) - wait_per_timeout = config(:wait_per_timeout) + jitter = :rand.uniform(config!(:max_jitter)) + wait_per_timeout = config!(:wait_per_timeout) wait_coefficient * (wait_per_timeout + jitter) end defp table do :rolling_window_opts - |> config() + |> config!() |> Keyword.fetch!(:table) end - defp config(key) do + defp throttle_table do + case config(:throttle_rolling_window_opts) do + nil -> :ignore + keyword -> Keyword.fetch!(keyword, :table) + end + end + + defp config!(key) do :ethereum_jsonrpc |> Application.get_env(__MODULE__) |> Keyword.fetch!(key) end + + defp config(key) do + :ethereum_jsonrpc + |> Application.get_env(__MODULE__) + |> Keyword.get(key) + end end diff --git a/apps/ethereum_jsonrpc/lib/rsk.ex b/apps/ethereum_jsonrpc/lib/rsk.ex new file mode 100644 index 0000000000..9772ca5c0a --- /dev/null +++ b/apps/ethereum_jsonrpc/lib/rsk.ex @@ -0,0 +1,12 @@ +defmodule EthereumJSONRPC.RSK do + @moduledoc """ + Ethereum JSONRPC methods that are/are not supported by [RSK](https://www.rsk.co/). + """ + + @behaviour EthereumJSONRPC.Variant + + def fetch_internal_transactions(_, _), do: :ignore + def fetch_pending_transactions(_), do: :ignore + def fetch_block_internal_transactions(_block_numbers, _json_rpc_named_arguments), do: :ignore + def fetch_beneficiaries(_, _), do: :ignore +end diff --git a/apps/ethereum_jsonrpc/test/ethereum_jsonrpc/request_coordinator_test.exs b/apps/ethereum_jsonrpc/test/ethereum_jsonrpc/request_coordinator_test.exs index d299491eca..efda05ab86 100644 --- a/apps/ethereum_jsonrpc/test/ethereum_jsonrpc/request_coordinator_test.exs +++ b/apps/ethereum_jsonrpc/test/ethereum_jsonrpc/request_coordinator_test.exs @@ -11,38 +11,50 @@ defmodule EthereumJSONRPC.RequestCoordinatorTest do setup :verify_on_exit! setup do - table = Application.get_env(:ethereum_jsonrpc, EthereumJSONRPC.RequestCoordinator)[:rolling_window_opts][:table] + timeout_table = + Application.get_env(:ethereum_jsonrpc, EthereumJSONRPC.RequestCoordinator)[:rolling_window_opts][:table] - :ets.delete_all_objects(table) + throttle_table = + Application.get_env(:ethereum_jsonrpc, EthereumJSONRPC.RequestCoordinator)[:throttle_rolling_window_opts][:table] - %{table: table} + :ets.delete_all_objects(timeout_table) + :ets.delete_all_objects(throttle_table) + + %{timeout_table: timeout_table, throttle_table: throttle_table} end describe "perform/4" do - test "forwards result whenever a request doesn't timeout", %{table: table} do + test "forwards result whenever a request doesn't timeout", %{timeout_table: timeout_table} do expect(EthereumJSONRPC.Mox, :json_rpc, fn _, _ -> {:ok, %{}} end) - assert RollingWindow.count(table, :throttleable_error_count) == 0 + assert RollingWindow.count(timeout_table, :throttleable_error_count) == 0 assert {:ok, %{}} == RequestCoordinator.perform(%{}, EthereumJSONRPC.Mox, [], :timer.minutes(60)) - assert RollingWindow.count(table, :throttleable_error_count) == 0 + assert RollingWindow.count(timeout_table, :throttleable_error_count) == 0 end - test "increments counter on certain errors", %{table: table} do + test "increments counter on certain errors", %{timeout_table: timeout_table} do expect(EthereumJSONRPC.Mox, :json_rpc, fn :timeout, _ -> {:error, :timeout} end) expect(EthereumJSONRPC.Mox, :json_rpc, fn :bad_gateway, _ -> {:error, {:bad_gateway, "message"}} end) assert {:error, :timeout} == RequestCoordinator.perform(:timeout, EthereumJSONRPC.Mox, [], :timer.minutes(60)) - assert RollingWindow.count(table, :throttleable_error_count) == 1 + assert RollingWindow.count(timeout_table, :throttleable_error_count) == 1 assert {:error, {:bad_gateway, "message"}} == RequestCoordinator.perform(:bad_gateway, EthereumJSONRPC.Mox, [], :timer.minutes(60)) - assert RollingWindow.count(table, :throttleable_error_count) == 2 + assert RollingWindow.count(timeout_table, :throttleable_error_count) == 2 end - test "returns timeout error if sleep time will exceed max timeout", %{table: table} do + test "returns timeout error if sleep time will exceed max timeout", %{timeout_table: timeout_table} do expect(EthereumJSONRPC.Mox, :json_rpc, 0, fn _, _ -> :ok end) - RollingWindow.inc(table, :throttleable_error_count) + RollingWindow.inc(timeout_table, :throttleable_error_count) assert {:error, :timeout} == RequestCoordinator.perform(%{}, EthereumJSONRPC.Mox, [], 1) end + + test "increments throttle_table even when not an error", %{throttle_table: throttle_table} do + expect(EthereumJSONRPC.Mox, :json_rpc, fn _, _ -> {:ok, %{}} end) + assert RollingWindow.count(throttle_table, :throttle_requests_count) == 0 + assert {:ok, %{}} == RequestCoordinator.perform(%{}, EthereumJSONRPC.Mox, [], :timer.minutes(60)) + assert RollingWindow.count(throttle_table, :throttle_requests_count) == 1 + end end end diff --git a/apps/explorer/config/dev/rsk.exs b/apps/explorer/config/dev/rsk.exs new file mode 100644 index 0000000000..3928520d30 --- /dev/null +++ b/apps/explorer/config/dev/rsk.exs @@ -0,0 +1,25 @@ +use Mix.Config + +config :explorer, + json_rpc_named_arguments: [ + transport: EthereumJSONRPC.HTTP, + transport_options: [ + http: EthereumJSONRPC.HTTP.HTTPoison, + url: System.get_env("ETHEREUM_JSONRPC_HTTP_URL") || "http://localhost:8545", + method_to_url: [ + eth_call: System.get_env("ETHEREUM_JSONRPC_TRACE_URL") || "http://localhost:8545", + eth_getBalance: System.get_env("ETHEREUM_JSONRPC_TRACE_URL") || "http://localhost:8545", + trace_replayTransaction: System.get_env("ETHEREUM_JSONRPC_TRACE_URL") || "http://localhost:8545" + ], + http_options: [recv_timeout: :timer.minutes(1), timeout: :timer.minutes(1), hackney: [pool: :ethereum_jsonrpc]] + ], + variant: EthereumJSONRPC.RSK + ], + subscribe_named_arguments: [ + transport: EthereumJSONRPC.WebSocket, + transport_options: [ + web_socket: EthereumJSONRPC.WebSocket.WebSocketClient, + url: System.get_env("ETHEREUM_JSONRPC_WS_URL") || "ws://localhost:8546" + ], + variant: EthereumJSONRPC.RSK + ] diff --git a/apps/explorer/config/prod/rsk.exs b/apps/explorer/config/prod/rsk.exs new file mode 100644 index 0000000000..fc62a1c431 --- /dev/null +++ b/apps/explorer/config/prod/rsk.exs @@ -0,0 +1,25 @@ +use Mix.Config + +config :explorer, + json_rpc_named_arguments: [ + transport: EthereumJSONRPC.HTTP, + transport_options: [ + http: EthereumJSONRPC.HTTP.HTTPoison, + url: System.get_env("ETHEREUM_JSONRPC_HTTP_URL"), + method_to_url: [ + eth_call: System.get_env("ETHEREUM_JSONRPC_TRACE_URL"), + eth_getBalance: System.get_env("ETHEREUM_JSONRPC_TRACE_URL"), + trace_replayTransaction: System.get_env("ETHEREUM_JSONRPC_TRACE_URL") + ], + http_options: [recv_timeout: :timer.minutes(1), timeout: :timer.minutes(1), hackney: [pool: :ethereum_jsonrpc]] + ], + variant: EthereumJSONRPC.RSK + ], + subscribe_named_arguments: [ + transport: EthereumJSONRPC.WebSocket, + transport_options: [ + web_socket: EthereumJSONRPC.WebSocket.WebSocketClient, + url: System.get_env("ETHEREUM_JSONRPC_WS_URL") + ], + variant: EthereumJSONRPC.RSK + ] diff --git a/apps/explorer/config/test/rsk.exs b/apps/explorer/config/test/rsk.exs new file mode 100644 index 0000000000..861525bb0d --- /dev/null +++ b/apps/explorer/config/test/rsk.exs @@ -0,0 +1,14 @@ +use Mix.Config + +config :explorer, + transport: EthereumJSONRPC.HTTP, + json_rpc_named_arguments: [ + transport: EthereumJSONRPC.Mox, + transport_options: [], + variant: EthereumJSONRPC.RSK + ], + subscribe_named_arguments: [ + transport: EthereumJSONRPC.Mox, + transport_options: [], + variant: EthereumJSONRPC.RSK + ] diff --git a/apps/indexer/config/dev/rsk.exs b/apps/indexer/config/dev/rsk.exs new file mode 100644 index 0000000000..21138634f5 --- /dev/null +++ b/apps/indexer/config/dev/rsk.exs @@ -0,0 +1,27 @@ +use Mix.Config + +config :indexer, + block_interval: :timer.seconds(5), + blocks_concurrency: 1, + receipts_concurrency: 1, + json_rpc_named_arguments: [ + transport: EthereumJSONRPC.HTTP, + transport_options: [ + http: EthereumJSONRPC.HTTP.HTTPoison, + url: System.get_env("ETHEREUM_JSONRPC_HTTP_URL") || "http://localhost:8545", + method_to_url: [ + eth_getBalance: System.get_env("ETHEREUM_JSONRPC_TRACE_URL") || "http://localhost:8545", + trace_block: System.get_env("ETHEREUM_JSONRPC_TRACE_URL") || "http://localhost:8545", + trace_replayBlockTransactions: System.get_env("ETHEREUM_JSONRPC_TRACE_URL") || "http://localhost:8545" + ], + http_options: [recv_timeout: :timer.minutes(10), timeout: :timer.minutes(10), hackney: [pool: :ethereum_jsonrpc]] + ], + variant: EthereumJSONRPC.RSK + ], + subscribe_named_arguments: [ + transport: EthereumJSONRPC.WebSocket, + transport_options: [ + web_socket: EthereumJSONRPC.WebSocket.WebSocketClient, + url: System.get_env("ETHEREUM_JSONRPC_WS_URL") || "ws://localhost:8546" + ] + ] diff --git a/apps/indexer/config/prod/rsk.exs b/apps/indexer/config/prod/rsk.exs new file mode 100644 index 0000000000..1eeedabb57 --- /dev/null +++ b/apps/indexer/config/prod/rsk.exs @@ -0,0 +1,27 @@ +use Mix.Config + +config :indexer, + block_interval: :timer.seconds(5), + blocks_concurrency: 1, + receipts_concurrency: 1, + json_rpc_named_arguments: [ + transport: EthereumJSONRPC.HTTP, + transport_options: [ + http: EthereumJSONRPC.HTTP.HTTPoison, + url: System.get_env("ETHEREUM_JSONRPC_HTTP_URL"), + method_to_url: [ + eth_getBalance: System.get_env("ETHEREUM_JSONRPC_TRACE_URL"), + trace_block: System.get_env("ETHEREUM_JSONRPC_TRACE_URL"), + trace_replayTransaction: System.get_env("ETHEREUM_JSONRPC_TRACE_URL") + ], + http_options: [recv_timeout: :timer.minutes(10), timeout: :timer.minutes(10), hackney: [pool: :ethereum_jsonrpc]] + ], + variant: EthereumJSONRPC.RSK + ], + subscribe_named_arguments: [ + transport: EthereumJSONRPC.WebSocket, + transport_options: [ + web_socket: EthereumJSONRPC.WebSocket.WebSocketClient, + url: System.get_env("ETHEREUM_JSONRPC_WS_URL") + ] + ] diff --git a/apps/indexer/lib/indexer/supervisor.ex b/apps/indexer/lib/indexer/supervisor.ex index 3626c31240..113aa56576 100644 --- a/apps/indexer/lib/indexer/supervisor.ex +++ b/apps/indexer/lib/indexer/supervisor.ex @@ -75,7 +75,7 @@ defmodule Indexer.Supervisor do block_fetcher = named_arguments - |> Map.drop(~w(block_interval memory_monitor subscribe_named_arguments realtime_overrides)a) + |> Map.drop(~w(block_interval blocks_concurrency memory_monitor subscribe_named_arguments realtime_overrides)a) |> Block.Fetcher.new() fixing_realtime_fetcher = %Block.Fetcher{ @@ -86,7 +86,7 @@ defmodule Indexer.Supervisor do realtime_block_fetcher = named_arguments - |> Map.drop(~w(block_interval memory_monitor subscribe_named_arguments realtime_overrides)a) + |> Map.drop(~w(block_interval blocks_concurrency memory_monitor subscribe_named_arguments realtime_overrides)a) |> Map.merge(Enum.into(realtime_overrides, %{})) |> Block.Fetcher.new()