API rate limit

pull/5030/head
Viktor Baranov 3 years ago
parent d30d49bdd8
commit 35f2df7c46
  1. 2
      .dialyzer-ignore
  2. 1
      CHANGELOG.md
  3. 4
      apps/block_scout_web/assets/js/lib/try_eth_api.js
  4. 14
      apps/block_scout_web/config/config.exs
  5. 8
      apps/block_scout_web/lib/block_scout_web/api_router.ex
  6. 10
      apps/block_scout_web/lib/block_scout_web/controllers/api/api_logger.ex
  7. 63
      apps/block_scout_web/lib/block_scout_web/controllers/api/eth_rpc/eth_controller.ex
  8. 40
      apps/block_scout_web/lib/block_scout_web/controllers/api/rpc/eth_controller.ex
  9. 11
      apps/block_scout_web/lib/block_scout_web/controllers/api/rpc/rpc_translator.ex
  10. 1
      apps/block_scout_web/lib/block_scout_web/controllers/api/rpc/transaction_controller.ex
  11. 3
      apps/block_scout_web/lib/block_scout_web/controllers/api/v1/decompiled_smart_contract_controller.ex
  12. 3
      apps/block_scout_web/lib/block_scout_web/controllers/api/v1/health_controller.ex
  13. 2
      apps/block_scout_web/lib/block_scout_web/controllers/api/v1/supply_controller.ex
  14. 3
      apps/block_scout_web/lib/block_scout_web/controllers/api/v1/verified_smart_contract_controller.ex
  15. 28
      apps/block_scout_web/lib/block_scout_web/views/access_helpers.ex
  16. 17
      apps/block_scout_web/lib/block_scout_web/views/api/eth_rpc/view.ex
  17. 3
      apps/block_scout_web/lib/block_scout_web/views/api/rpc/address_view.ex
  18. 3
      apps/block_scout_web/lib/block_scout_web/views/api/rpc/block_view.ex
  19. 13
      apps/block_scout_web/lib/block_scout_web/views/api/rpc/eth_view.ex
  20. 1
      apps/block_scout_web/mix.exs
  21. 3
      docker/Makefile
  22. 1
      mix.lock

@ -17,7 +17,7 @@ lib/phoenix/router.ex:324
lib/phoenix/router.ex:402
lib/block_scout_web/views/layout_view.ex:145: The call 'Elixir.Poison.Parser':'parse!'
lib/block_scout_web/views/layout_view.ex:237: The call 'Elixir.Poison.Parser':'parse!'
lib/block_scout_web/controllers/api/rpc/transaction_controller.ex:23
lib/block_scout_web/controllers/api/rpc/transaction_controller.ex:22
lib/explorer/smart_contract/reader.ex:461
lib/indexer/fetcher/token_total_supply_on_demand.ex:16
lib/explorer/exchange_rates/source.ex:110

@ -1,6 +1,7 @@
## Current
### Features
- [#5030](https://github.com/blockscout/blockscout/pull/5030) - API rate limiting
- [#4924](https://github.com/blockscout/blockscout/pull/4924) - Add daily bytecode verifcation to prevent metamorphic contracts vulnerablity
- [#4908](https://github.com/blockscout/blockscout/pull/4908) - Add verification via standard JSON input
- [#5004](https://github.com/blockscout/blockscout/pull/5004) - Add ability to set up a separate DB endpoint for the API endpoints

@ -76,5 +76,7 @@ $('button[data-try-eth-api-ui-button-type="execute"]').click(event => {
data: JSON.stringify(formData),
dataType: 'json',
contentType: 'application/json; charset=utf-8'
}).then((_data, _status, xhr) => handleResponse(formData, xhr, clickedButton))
})
.then((_data, _status, xhr) => handleResponse(formData, xhr, clickedButton))
.fail((xhr) => handleResponse(formData, xhr, clickedButton))
})

@ -54,6 +54,17 @@ config :block_scout_web,
re_captcha_secret_key: System.get_env("RE_CAPTCHA_SECRET_KEY", nil),
re_captcha_client_key: System.get_env("RE_CAPTCHA_CLIENT_KEY", nil)
api_rate_limit_value =
"API_RATE_LIMIT"
|> System.get_env("30")
|> Integer.parse()
|> case do
{integer, ""} -> integer
_ -> 30
end
config :block_scout_web, api_rate_limit: api_rate_limit_value
config :block_scout_web, BlockScoutWeb.Counters.BlocksIndexedCounter, enabled: true
# Configures the endpoint
@ -143,6 +154,9 @@ config :block_scout_web, BlockScoutWeb.ApiRouter,
config :block_scout_web, BlockScoutWeb.WebRouter, enabled: System.get_env("DISABLE_WEBAPP") != "true"
config :hammer,
backend: {Hammer.Backend.ETS, [expiry_ms: 60_000 * 60 * 4, cleanup_interval_ms: 60_000 * 10]}
# Import environment specific config. This must remain at the bottom
# of this file so it overrides the configuration defined above.
import_config "#{Mix.env()}.exs"

@ -20,14 +20,14 @@ defmodule BlockScoutWeb.ApiRouter do
scope "/v1", as: :api_v1 do
pipe_through(:api)
alias BlockScoutWeb.API.{RPC, V1}
alias BlockScoutWeb.API.{EthRPC, RPC, V1}
alias BlockScoutWeb.API.V1.HealthController
get("/health", HealthController, :health)
if Application.get_env(:block_scout_web, __MODULE__)[:reading_enabled] do
get("/supply", V1.SupplyController, :supply)
post("/eth-rpc", RPC.EthController, :eth_request)
post("/eth-rpc", EthRPC.EthController, :eth_request)
end
if Application.get_env(:block_scout_web, __MODULE__)[:writing_enabled] do
@ -51,10 +51,10 @@ defmodule BlockScoutWeb.ApiRouter do
# For backward compatibility. Should be removed
scope "/" do
pipe_through(:api)
alias BlockScoutWeb.API.RPC
alias BlockScoutWeb.API.{EthRPC, RPC}
if Application.get_env(:block_scout_web, __MODULE__)[:reading_enabled] do
post("/eth-rpc", RPC.EthController, :eth_request)
post("/eth-rpc", EthRPC.EthController, :eth_request)
forward("/", RPCTranslatorForwarder, %{
"block" => {RPC.BlockController, []},

@ -1,6 +1,6 @@
defmodule APILogger do
defmodule BlockScoutWeb.API.APILogger do
@moduledoc """
Logger of API ednpoins usage
Logger for API endpoints usage
"""
require Logger
@ -16,4 +16,10 @@ defmodule APILogger do
fetcher: :api
)
end
def message(text) do
Logger.debug(text,
fetcher: :api
)
end
end

@ -0,0 +1,63 @@
defmodule BlockScoutWeb.API.EthRPC.EthController do
use BlockScoutWeb, :controller
alias BlockScoutWeb.AccessHelpers
alias BlockScoutWeb.API.EthRPC.View, as: EthRPCView
alias Explorer.EthRPC
def eth_request(%{body_params: %{"_json" => requests}} = conn, _) when is_list(requests) do
case AccessHelpers.check_rate_limit(conn) do
:ok ->
responses = EthRPC.responses(requests)
conn
|> put_status(200)
|> put_view(EthRPCView)
|> render("responses.json", %{responses: responses})
:rate_limit_reached ->
AccessHelpers.handle_rate_limit_deny(conn)
end
end
def eth_request(%{body_params: %{"_json" => request}} = conn, _) do
case AccessHelpers.check_rate_limit(conn) do
:ok ->
[response] = EthRPC.responses([request])
conn
|> put_status(200)
|> put_view(EthRPCView)
|> render("response.json", %{response: response})
:rate_limit_reached ->
AccessHelpers.handle_rate_limit_deny(conn)
end
end
def eth_request(conn, request) do
case AccessHelpers.check_rate_limit(conn) do
:ok ->
# In the case that the JSON body is sent up w/o a json content type,
# Phoenix encodes it as a single key value pair, with the value being
# nil and the body being the key (as in a CURL request w/ no content type header)
decoded_request =
with [{single_key, nil}] <- Map.to_list(request),
{:ok, decoded} <- Jason.decode(single_key) do
decoded
else
_ -> request
end
[response] = EthRPC.responses([decoded_request])
conn
|> put_status(200)
|> put_view(EthRPCView)
|> render("response.json", %{response: response})
:rate_limit_reached ->
AccessHelpers.handle_rate_limit_deny(conn)
end
end
end

@ -1,40 +0,0 @@
defmodule BlockScoutWeb.API.RPC.EthController do
use BlockScoutWeb, :controller
alias Explorer.EthRPC
def eth_request(%{body_params: %{"_json" => requests}} = conn, _) when is_list(requests) do
responses = EthRPC.responses(requests)
conn
|> put_status(200)
|> render("responses.json", %{responses: responses})
end
def eth_request(%{body_params: %{"_json" => request}} = conn, _) do
[response] = EthRPC.responses([request])
conn
|> put_status(200)
|> render("response.json", %{response: response})
end
def eth_request(conn, request) do
# In the case that the JSON body is sent up w/o a json content type,
# Phoenix encodes it as a single key value pair, with the value being
# nil and the body being the key (as in a CURL request w/ no content type header)
decoded_request =
with [{single_key, nil}] <- Map.to_list(request),
{:ok, decoded} <- Jason.decode(single_key) do
decoded
else
_ -> request
end
[response] = EthRPC.responses([decoded_request])
conn
|> put_status(200)
|> render("response.json", %{response: response})
end
end

@ -14,15 +14,20 @@ defmodule BlockScoutWeb.API.RPC.RPCTranslator do
"""
require Logger
require APILogger
import Plug.Conn
import Phoenix.Controller, only: [put_view: 2]
alias BlockScoutWeb.AccessHelpers
alias BlockScoutWeb.API.APILogger
alias BlockScoutWeb.API.RPC.RPCView
alias Phoenix.Controller
alias Plug.Conn
APILogger.message(
"Current API rate limit #{inspect(Application.get_env(:block_scout_web, :api_rate_limit))} reqs/sec"
)
def init(opts), do: opts
def call(%Conn{params: %{"module" => module, "action" => action}} = conn, translations) do
@ -30,6 +35,7 @@ defmodule BlockScoutWeb.API.RPC.RPCTranslator do
{:ok, {controller, write_actions}} <- translate_module(translations, module),
{:ok, action} <- translate_action(action),
true <- action_accessed?(action, write_actions),
:ok <- AccessHelpers.check_rate_limit(conn),
{:ok, conn} <- call_controller(conn, controller, action) do
APILogger.log(conn)
conn
@ -50,6 +56,9 @@ defmodule BlockScoutWeb.API.RPC.RPCTranslator do
|> Controller.render(:error, error: "Something went wrong.")
|> halt()
:rate_limit_reached ->
AccessHelpers.handle_rate_limit_deny(conn)
_ ->
conn
|> put_status(500)

@ -4,7 +4,6 @@ defmodule BlockScoutWeb.API.RPC.TransactionController do
import BlockScoutWeb.Chain, only: [paging_options: 1, next_page_params: 3, split_list_by_page: 1]
alias Explorer.Chain
alias Explorer.Chain.Transaction
def gettxinfo(conn, params) do

@ -1,8 +1,7 @@
defmodule BlockScoutWeb.API.V1.DecompiledSmartContractController do
use BlockScoutWeb, :controller
require APILogger
alias BlockScoutWeb.API.APILogger
alias Explorer.Chain
alias Explorer.Chain.Hash.Address

@ -1,8 +1,7 @@
defmodule BlockScoutWeb.API.V1.HealthController do
use BlockScoutWeb, :controller
require APILogger
alias BlockScoutWeb.API.APILogger
alias Explorer.Chain
def health(conn, _) do

@ -1,7 +1,7 @@
defmodule BlockScoutWeb.API.V1.SupplyController do
use BlockScoutWeb, :controller
require APILogger
alias BlockScoutWeb.API.APILogger
alias Explorer.Chain
def supply(conn, _) do

@ -1,8 +1,7 @@
defmodule BlockScoutWeb.API.V1.VerifiedSmartContractController do
use BlockScoutWeb, :controller
require APILogger
alias BlockScoutWeb.API.APILogger
alias Explorer.Chain
alias Explorer.Chain.Hash.Address
alias Explorer.SmartContract.Solidity.Publisher

@ -3,6 +3,10 @@ defmodule BlockScoutWeb.AccessHelpers do
Helpers to restrict access to some pages filtering by address
"""
import Phoenix.Controller
alias BlockScoutWeb.API.APILogger
alias BlockScoutWeb.API.RPC.RPCView
alias BlockScoutWeb.WebRouter.Helpers
alias Plug.Conn
@ -59,4 +63,28 @@ defmodule BlockScoutWeb.AccessHelpers do
apply(Helpers, path, full_args)
end
def handle_rate_limit_deny(conn) do
APILogger.message("API rate limit reached")
conn
|> Conn.put_status(429)
|> put_view(RPCView)
|> render(:error, %{error: "429 Too Many Requests"})
|> Conn.halt()
end
def check_rate_limit(_conn) do
if Mix.env() == :test do
:ok
else
case Hammer.check_rate("api", 1_000, Application.get_env(:block_scout_web, :api_rate_limit)) do
{:allow, _count} ->
:ok
{:deny, _limit} ->
:rate_limit_reached
end
end
end
end

@ -1,4 +1,7 @@
defmodule BlockScoutWeb.API.RPC.EthRPCView do
defmodule BlockScoutWeb.API.EthRPC.View do
@moduledoc """
Views for /eth-rpc API endpoints
"""
use BlockScoutWeb, :view
defstruct [:result, :id, :error]
@ -47,8 +50,8 @@ defmodule BlockScoutWeb.API.RPC.EthRPCView do
end)
end
defimpl Poison.Encoder, for: BlockScoutWeb.API.RPC.EthRPCView do
def encode(%BlockScoutWeb.API.RPC.EthRPCView{result: result, id: id, error: error}, _options) when is_nil(error) do
defimpl Poison.Encoder, for: BlockScoutWeb.API.EthRPC.View do
def encode(%BlockScoutWeb.API.EthRPC.View{result: result, id: id, error: error}, _options) when is_nil(error) do
result = Poison.encode!(result)
"""
@ -56,15 +59,15 @@ defmodule BlockScoutWeb.API.RPC.EthRPCView do
"""
end
def encode(%BlockScoutWeb.API.RPC.EthRPCView{id: id, error: error}, _options) do
def encode(%BlockScoutWeb.API.EthRPC.View{id: id, error: error}, _options) do
"""
{"jsonrpc":"2.0","error": "#{error}","id": #{id}}
"""
end
end
defimpl Jason.Encoder, for: BlockScoutWeb.API.RPC.EthRPCView do
def encode(%BlockScoutWeb.API.RPC.EthRPCView{result: result, id: id, error: error}, _options) when is_nil(error) do
defimpl Jason.Encoder, for: BlockScoutWeb.API.EthRPC.View do
def encode(%BlockScoutWeb.API.EthRPC.View{result: result, id: id, error: error}, _options) when is_nil(error) do
result = Jason.encode!(result)
"""
@ -72,7 +75,7 @@ defmodule BlockScoutWeb.API.RPC.EthRPCView do
"""
end
def encode(%BlockScoutWeb.API.RPC.EthRPCView{id: id, error: error}, _options) do
def encode(%BlockScoutWeb.API.EthRPC.View{id: id, error: error}, _options) do
"""
{"jsonrpc":"2.0","error": "#{error}","id": #{id}}
"""

@ -1,7 +1,8 @@
defmodule BlockScoutWeb.API.RPC.AddressView do
use BlockScoutWeb, :view
alias BlockScoutWeb.API.RPC.{EthRPCView, RPCView}
alias BlockScoutWeb.API.EthRPC.View, as: EthRPCView
alias BlockScoutWeb.API.RPC.RPCView
def render("listaccounts.json", %{accounts: accounts}) do
accounts = Enum.map(accounts, &prepare_account/1)

@ -1,7 +1,8 @@
defmodule BlockScoutWeb.API.RPC.BlockView do
use BlockScoutWeb, :view
alias BlockScoutWeb.API.RPC.{EthRPCView, RPCView}
alias BlockScoutWeb.API.EthRPC.View, as: EthRPCView
alias BlockScoutWeb.API.RPC.RPCView
alias Explorer.Chain.{Hash, Wei}
alias Explorer.EthRPC, as: EthRPC

@ -1,13 +0,0 @@
defmodule BlockScoutWeb.API.RPC.EthView do
use BlockScoutWeb, :view
alias BlockScoutWeb.API.RPC.EthRPCView
def render("responses.json", %{responses: responses}) do
EthRPCView.render("responses.json", %{responses: responses})
end
def render("response.json", %{response: response}) do
EthRPCView.render("response.json", %{response: response})
end
end

@ -84,6 +84,7 @@ defmodule BlockScoutWeb.Mixfile do
{:floki, "~> 0.31"},
{:flow, "~> 0.12"},
{:gettext, "~> 0.18.2"},
{:hammer, "~> 6.0"},
{:httpoison, "~> 1.6"},
{:indexer, in_umbrella: true, runtime: false},
# JSON parser and generator

@ -353,6 +353,9 @@ endif
ifdef ADDRESS_TOKEN_TRANSFERS_COUNTER_CACHE_PERIOD
BLOCKSCOUT_CONTAINER_PARAMS += -e 'ADDRESS_TOKEN_TRANSFERS_COUNTER_CACHE_PERIOD=$(ADDRESS_TOKEN_TRANSFERS_COUNTER_CACHE_PERIOD)'
endif
ifdef API_RATE_LIMIT
BLOCKSCOUT_CONTAINER_PARAMS += -e 'API_RATE_LIMIT=$(API_RATE_LIMIT)'
endif
HAS_BLOCKSCOUT_IMAGE := $(shell docker images | grep -sw ${DOCKER_IMAGE})
build:

@ -57,6 +57,7 @@
"gen_stage": {:hex, :gen_stage, "0.14.3", "d0c66f1c87faa301c1a85a809a3ee9097a4264b2edf7644bf5c123237ef732bf", [:mix], [], "hexpm", "8453e2289d94c3199396eb517d65d6715ef26bcae0ee83eb5ff7a84445458d76"},
"gettext": {:hex, :gettext, "0.18.2", "7df3ea191bb56c0309c00a783334b288d08a879f53a7014341284635850a6e55", [:mix], [], "hexpm", "f9f537b13d4fdd30f3039d33cb80144c3aa1f8d9698e47d7bcbcc8df93b1f5c5"},
"hackney": {:hex, :hackney, "1.17.4", "99da4674592504d3fb0cfef0db84c3ba02b4508bae2dff8c0108baa0d6e0977c", [:rebar3], [{:certifi, "~>2.6.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~>6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~>1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.3.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~>1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "de16ff4996556c8548d512f4dbe22dd58a587bf3332e7fd362430a7ef3986b16"},
"hammer": {:hex, :hammer, "6.0.0", "72ec6fff10e9d63856968988a22ee04c4d6d5248071ddccfbda50aa6c455c1d7", [:mix], [{:poolboy, "~> 1.5", [hex: :poolboy, repo: "hexpm", optional: false]}], "hexpm", "d8e1ec2e534c4aae508b906759e077c3c1eb3e2b9425235d4b7bbab0b016210a"},
"html_entities": {:hex, :html_entities, "0.5.2", "9e47e70598da7de2a9ff6af8758399251db6dbb7eebe2b013f2bbd2515895c3c", [:mix], [], "hexpm", "c53ba390403485615623b9531e97696f076ed415e8d8058b1dbaa28181f4fdcc"},
"httpoison": {:hex, :httpoison, "1.8.0", "6b85dea15820b7804ef607ff78406ab449dd78bed923a49c7160e1886e987a3d", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "28089eaa98cf90c66265b6b5ad87c59a3729bea2e74e9d08f9b51eb9729b3c3a"},
"idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"},

Loading…
Cancel
Save