From 4ad36e2f79b3f7ea250e4f331b0835ad4699e928 Mon Sep 17 00:00:00 2001 From: Viktor Baranov Date: Wed, 12 Jan 2022 17:56:20 +0300 Subject: [PATCH] Dedicated rate limiting by IP --- CHANGELOG.md | 2 + apps/block_scout_web/config/config.exs | 19 +++- .../controllers/api/rpc/rpc_translator.ex | 8 +- .../block_scout_web/views/access_helpers.ex | 103 +++++++++++++----- apps/block_scout_web/mix.exs | 1 + mix.lock | 1 + 6 files changed, 99 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c383f570d..b239b1b3b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ ## Current ### Features +- [#5090](https://github.com/blockscout/blockscout/pull/5090) - Allotted rate limit by IP +- [#5080](https://github.com/blockscout/blockscout/pull/5080) - Alloted rate limit by a global API key ### Fixes - [#5088](https://github.com/blockscout/blockscout/pull/5088) - Store address transactions/token transfers in the DB diff --git a/apps/block_scout_web/config/config.exs b/apps/block_scout_web/config/config.exs index 0a66411ac0..836b27814e 100644 --- a/apps/block_scout_web/config/config.exs +++ b/apps/block_scout_web/config/config.exs @@ -72,10 +72,21 @@ api_rate_limit_by_key_value = _ -> 50 end -config :block_scout_web, - global_api_rate_limit: global_api_rate_limit_value, - api_rate_limit_by_key: api_rate_limit_by_key_value, - static_api_key: System.get_env("STATIC_API_KEY", nil) +api_rate_limit_by_ip_value = + "API_RATE_LIMIT_BY_IP" + |> System.get_env("50") + |> Integer.parse() + |> case do + {integer, ""} -> integer + _ -> 50 + end + +config :block_scout_web, :api_rate_limit, + global_limit: global_api_rate_limit_value, + limit_by_key: api_rate_limit_by_key_value, + limit_by_ip: api_rate_limit_by_ip_value, + static_api_key: System.get_env("API_RATE_LIMIT_STATIC_API_KEY", nil), + whitelisted_ips: System.get_env("API_RATE_LIMIT_WHITELISTED_IPS", nil) config :block_scout_web, BlockScoutWeb.Counters.BlocksIndexedCounter, enabled: true diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/api/rpc/rpc_translator.ex b/apps/block_scout_web/lib/block_scout_web/controllers/api/rpc/rpc_translator.ex index 7c471a2bff..7471209b9a 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/api/rpc/rpc_translator.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/api/rpc/rpc_translator.ex @@ -25,11 +25,15 @@ defmodule BlockScoutWeb.API.RPC.RPCTranslator do alias Plug.Conn APILogger.message( - "Current global API rate limit #{inspect(Application.get_env(:block_scout_web, :global_api_rate_limit))} reqs/sec" + "Current global API rate limit #{inspect(Application.get_env(:block_scout_web, :api_rate_limit)[:global_limit])} reqs/sec" ) APILogger.message( - "Current API rate limit by key #{inspect(Application.get_env(:block_scout_web, :api_rate_limit_by_key))} reqs/sec" + "Current API rate limit by key #{inspect(Application.get_env(:block_scout_web, :api_rate_limit)[:limit_by_key])} reqs/sec" + ) + + APILogger.message( + "Current API rate limit by IP #{inspect(Application.get_env(:block_scout_web, :api_rate_limit)[:limit_by_ip])} reqs/sec" ) def init(opts), do: opts diff --git a/apps/block_scout_web/lib/block_scout_web/views/access_helpers.ex b/apps/block_scout_web/lib/block_scout_web/views/access_helpers.ex index 1d1c9da220..c89ac10435 100644 --- a/apps/block_scout_web/lib/block_scout_web/views/access_helpers.ex +++ b/apps/block_scout_web/lib/block_scout_web/views/access_helpers.ex @@ -10,14 +10,7 @@ defmodule BlockScoutWeb.AccessHelpers do alias BlockScoutWeb.WebRouter.Helpers alias Plug.Conn - defp get_restricted_key(%Phoenix.Socket{}) do - nil - end - - defp get_restricted_key(conn) do - conn_with_params = Conn.fetch_query_params(conn) - conn_with_params.query_params["key"] - end + alias RemoteIp def restricted_access?(address_hash, params) do restricted_list_var = Application.get_env(:block_scout_web, :restricted_list) @@ -78,28 +71,80 @@ defmodule BlockScoutWeb.AccessHelpers do if Mix.env() == :test do :ok else - global_api_rate_limit = Application.get_env(:block_scout_web, :global_api_rate_limit) - api_rate_limit_by_key = Application.get_env(:block_scout_web, :api_rate_limit_by_key) - static_api_key = Application.get_env(:block_scout_web, :static_api_key) - - if conn.query_params && Map.has_key?(conn.query_params, "apikey") && - Map.get(conn.query_params, "apikey") == static_api_key do - case Hammer.check_rate("api-#{static_api_key}", 1_000, api_rate_limit_by_key) do - {:allow, _count} -> - :ok - - {:deny, _limit} -> - :rate_limit_reached - end - else - case Hammer.check_rate("api", 1_000, global_api_rate_limit) do - {:allow, _count} -> - :ok - - {:deny, _limit} -> - :rate_limit_reached - end + global_api_rate_limit = Application.get_env(:block_scout_web, :api_rate_limit)[:global_limit] + api_rate_limit_by_key = Application.get_env(:block_scout_web, :api_rate_limit)[:api_rate_limit_by_key] + api_rate_limit_by_ip = Application.get_env(:block_scout_web, :api_rate_limit)[:limit_by_ip] + static_api_key = Application.get_env(:block_scout_web, :api_rate_limit)[:static_api_key] + + remote_ip = conn.remote_ip + remote_ip_from_headers = RemoteIp.from(conn.resp_headers) + ip = remote_ip_from_headers || remote_ip + ip_string = to_string(:inet_parse.ntoa(ip)) + + cond do + conn.query_params && Map.has_key?(conn.query_params, "apikey") && + Map.get(conn.query_params, "apikey") == static_api_key -> + rate_limit_by_key(static_api_key, api_rate_limit_by_key) + + Enum.member?(api_rate_limit_whitelisted_ips(), ip_string) -> + rate_limit_by_ip(ip_string, api_rate_limit_by_ip) + + true -> + global_rate_limit(global_api_rate_limit) end end end + + defp rate_limit_by_key(api_key, api_rate_limit_by_key) do + case Hammer.check_rate("api-#{api_key}", 1_000, api_rate_limit_by_key) do + {:allow, _count} -> + :ok + + {:deny, _limit} -> + :rate_limit_reached + end + end + + defp rate_limit_by_ip(ip_string, api_rate_limit_by_ip) do + case Hammer.check_rate("api-#{ip_string}", 1_000, api_rate_limit_by_ip) do + {:allow, _count} -> + :ok + + {:deny, _limit} -> + :rate_limit_reached + end + end + + defp global_rate_limit(global_api_rate_limit) do + case Hammer.check_rate("api", 1_000, global_api_rate_limit) do + {:allow, _count} -> + :ok + + {:deny, _limit} -> + :rate_limit_reached + end + end + + defp get_restricted_key(%Phoenix.Socket{}) do + nil + end + + defp get_restricted_key(conn) do + conn_with_params = Conn.fetch_query_params(conn) + conn_with_params.query_params["key"] + end + + defp api_rate_limit_whitelisted_ips do + with api_rate_limit_object <- + :block_scout_web + |> Application.get_env(:api_rate_limit), + {:ok, whitelisted_ips_string} <- + api_rate_limit_object && + api_rate_limit_object + |> Keyword.fetch(:whitelisted_ips) do + if whitelisted_ips_string, do: String.split(whitelisted_ips_string, ","), else: [] + else + _ -> [] + end + end end diff --git a/apps/block_scout_web/mix.exs b/apps/block_scout_web/mix.exs index d3f381b0a9..0bc11d48f8 100644 --- a/apps/block_scout_web/mix.exs +++ b/apps/block_scout_web/mix.exs @@ -114,6 +114,7 @@ defmodule BlockScoutWeb.Mixfile do {:prometheus_plugs, "~> 1.1"}, # OS process metrics for Prometheus {:prometheus_process_collector, "~> 1.3"}, + {:remote_ip, "~> 1.0"}, {:qrcode, "~> 0.1.0"}, {:sobelow, ">= 0.7.0", only: [:dev, :test], runtime: false}, # Tracing diff --git a/mix.lock b/mix.lock index 79a13f47ed..4598ad9168 100644 --- a/mix.lock +++ b/mix.lock @@ -108,6 +108,7 @@ "quantile_estimator": {:hex, :quantile_estimator, "0.2.1", "ef50a361f11b5f26b5f16d0696e46a9e4661756492c981f7b2229ef42ff1cd15", [:rebar3], [], "hexpm", "282a8a323ca2a845c9e6f787d166348f776c1d4a41ede63046d72d422e3da946"}, "que": {:hex, :que, "0.10.1", "788ed0ec92ed69bdf9cfb29bf41a94ca6355b8d44959bd0669cf706e557ac891", [:mix], [{:ex_utils, "~> 0.1.6", [hex: :ex_utils, repo: "hexpm", optional: false]}, {:memento, "~> 0.3.0", [hex: :memento, repo: "hexpm", optional: false]}], "hexpm", "a737b365253e75dbd24b2d51acc1d851049e87baae08cd0c94e2bc5cd65088d5"}, "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, + "remote_ip": {:hex, :remote_ip, "1.0.0", "3d7fb45204a5704443f480cee9515e464997f52c35e0a60b6ece1f81484067ae", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:plug, "~> 1.10", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "9e9fcad4e50c43b5234bb6a9629ed6ab223f3ed07147bd35470e4ee5c8caf907"}, "rustler": {:hex, :rustler, "0.23.0", "87162ffdf5a46b6aa03d624a77367070ff1263961ae35332c059225e136c4a87", [:mix], [{:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: false]}, {:toml, "~> 0.5.2", [hex: :toml, repo: "hexpm", optional: false]}], "hexpm", "f5ab6f0ec564f5569009c0f5685b0e5b379fd72655e82a8dc5a3c24f9fdda36a"}, "sobelow": {:hex, :sobelow, "0.11.1", "23438964486f8112b41e743bbfd402da3e5b296fdc9eacab29914b79c48916dd", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "9897363a7eff96f4809304a90aad819e2ad5e5d24db547af502885146746a53c"}, "spandex": {:hex, :spandex, "3.0.3", "91aa318f3de696bb4d931adf65f7ebdbe5df25cccce1fe8fd376a44c46bcf69b", [:mix], [{:decorator, "~> 1.2", [hex: :decorator, repo: "hexpm", optional: true]}, {:optimal, "~> 0.3.3", [hex: :optimal, repo: "hexpm", optional: false]}, {:plug, ">= 1.0.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "e3e6c319d0ab478ddc9a39102a727a410c962b4d51c0932c72279b86d3b17044"},