From 0f153faf944fa3a0a84c8a7a5661c6b0e1988367 Mon Sep 17 00:00:00 2001 From: nikitosing <32202610+nikitosing@users.noreply.github.com> Date: Wed, 3 Apr 2024 20:12:46 +0300 Subject: [PATCH] feat: Add rate limits to graphQL API (#9771) * feat: Add rate limits to graphQL API * Fix tests * Process review comments * Update common-blockscout.env --- .github/workflows/config.yml | 1 + .../lib/block_scout_web/api_router.ex | 28 ++++++++++----- .../lib/block_scout_web/plug/rate_limit.ex | 13 +++++++ .../lib/block_scout_web/router.ex | 28 ++++++++++----- .../block_scout_web/views/access_helper.ex | 36 +++++++++++++++++++ config/runtime.exs | 11 +++++- docker-compose/envs/common-blockscout.env | 12 ++++++- 7 files changed, 109 insertions(+), 20 deletions(-) diff --git a/.github/workflows/config.yml b/.github/workflows/config.yml index e3ee16f033..a7f759e304 100644 --- a/.github/workflows/config.yml +++ b/.github/workflows/config.yml @@ -743,6 +743,7 @@ jobs: ETHEREUM_JSONRPC_WEB_SOCKET_CASE: "EthereumJSONRPC.WebSocket.Case.Mox" CHAIN_ID: "10200" API_RATE_LIMIT_DISABLED: "true" + API_GRAPHQL_RATE_LIMIT_DISABLED: "true" ADMIN_PANEL_ENABLED: "true" ACCOUNT_ENABLED: "true" ACCOUNT_REDIS_URL: "redis://localhost:6379" diff --git a/apps/block_scout_web/lib/block_scout_web/api_router.ex b/apps/block_scout_web/lib/block_scout_web/api_router.ex index 19f4f1f799..e609de0fda 100644 --- a/apps/block_scout_web/lib/block_scout_web/api_router.ex +++ b/apps/block_scout_web/lib/block_scout_web/api_router.ex @@ -47,6 +47,12 @@ defmodule BlockScoutWeb.ApiRouter do plug(RateLimit) end + pipeline :api_v1_graphql do + plug(BlockScoutWeb.Plug.Logger, application: :api) + plug(:accepts, ["json"]) + plug(RateLimit, graphql?: true) + end + alias BlockScoutWeb.Account.Api.V2.{AuthenticateController, EmailController, TagsController, UserController} alias BlockScoutWeb.API.V2 @@ -336,23 +342,27 @@ defmodule BlockScoutWeb.ApiRouter do end end - scope "/v1", as: :api_v1 do - pipe_through(:api) - alias BlockScoutWeb.API.{EthRPC, RPC, V1} - alias BlockScoutWeb.API.V1.{GasPriceOracleController, HealthController} - alias BlockScoutWeb.API.V2.SearchController - - # leave the same endpoint in v1 in order to keep backward compatibility - get("/search", SearchController, :search) + scope "/v1/graphql" do + pipe_through(:api_v1_graphql) if Application.compile_env(:block_scout_web, Api.GraphQL)[:enabled] do - forward("/graphql", Absinthe.Plug, + forward("/", Absinthe.Plug, schema: BlockScoutWeb.GraphQL.Schema, analyze_complexity: true, max_complexity: Application.compile_env(:block_scout_web, Api.GraphQL)[:max_complexity], token_limit: Application.compile_env(:block_scout_web, Api.GraphQL)[:token_limit] ) end + end + + scope "/v1", as: :api_v1 do + pipe_through(:api) + alias BlockScoutWeb.API.{EthRPC, RPC, V1} + alias BlockScoutWeb.API.V1.{GasPriceOracleController, HealthController} + alias BlockScoutWeb.API.V2.SearchController + + # leave the same endpoint in v1 in order to keep backward compatibility + get("/search", SearchController, :search) get("/transactions-csv", AddressTransactionController, :transactions_csv) diff --git a/apps/block_scout_web/lib/block_scout_web/plug/rate_limit.ex b/apps/block_scout_web/lib/block_scout_web/plug/rate_limit.ex index c7e22f4e9c..5f6de5f4fe 100644 --- a/apps/block_scout_web/lib/block_scout_web/plug/rate_limit.ex +++ b/apps/block_scout_web/lib/block_scout_web/plug/rate_limit.ex @@ -6,6 +6,19 @@ defmodule BlockScoutWeb.Plug.RateLimit do def init(opts), do: opts + def call(conn, graphql?: true) do + case AccessHelper.check_rate_limit(conn, graphql?: true) do + :ok -> + conn + + true -> + conn + + _ -> + AccessHelper.handle_rate_limit_deny(conn, true) + end + end + def call(conn, _opts) do case AccessHelper.check_rate_limit(conn) do :ok -> diff --git a/apps/block_scout_web/lib/block_scout_web/router.ex b/apps/block_scout_web/lib/block_scout_web/router.ex index e580e167be..86da395f13 100644 --- a/apps/block_scout_web/lib/block_scout_web/router.ex +++ b/apps/block_scout_web/lib/block_scout_web/router.ex @@ -1,7 +1,7 @@ defmodule BlockScoutWeb.Router do use BlockScoutWeb, :router - alias BlockScoutWeb.Plug.GraphQL + alias BlockScoutWeb.Plug.{GraphQL, RateLimit} alias BlockScoutWeb.{ApiRouter, WebRouter} if Application.compile_env(:block_scout_web, :admin_panel_enabled) do @@ -22,16 +22,26 @@ defmodule BlockScoutWeb.Router do plug(:accepts, ["json"]) end + pipeline :api_v1_graphql do + plug(BlockScoutWeb.Plug.Logger, application: :api) + plug(:accepts, ["json"]) + plug(RateLimit, graphql?: true) + end + forward("/api", ApiRouter) - if Application.compile_env(:block_scout_web, Api.GraphQL)[:enabled] && - Application.compile_env(:block_scout_web, ApiRouter)[:reading_enabled] do - forward("/graphiql", Absinthe.Plug.GraphiQL, - schema: BlockScoutWeb.GraphQL.Schema, - interface: :advanced, - default_query: GraphQL.default_query(), - socket: BlockScoutWeb.UserSocket - ) + scope "/graphiql" do + pipe_through(:api_v1_graphql) + + if Application.compile_env(:block_scout_web, Api.GraphQL)[:enabled] && + Application.compile_env(:block_scout_web, ApiRouter)[:reading_enabled] do + forward("/", Absinthe.Plug.GraphiQL, + schema: BlockScoutWeb.GraphQL.Schema, + interface: :advanced, + default_query: GraphQL.default_query(), + socket: BlockScoutWeb.UserSocket + ) + end end scope "/", BlockScoutWeb do diff --git a/apps/block_scout_web/lib/block_scout_web/views/access_helper.ex b/apps/block_scout_web/lib/block_scout_web/views/access_helper.ex index ecfb0ffe55..8d200b36de 100644 --- a/apps/block_scout_web/lib/block_scout_web/views/access_helper.ex +++ b/apps/block_scout_web/lib/block_scout_web/views/access_helper.ex @@ -53,6 +53,42 @@ defmodule BlockScoutWeb.AccessHelper do |> Conn.halt() end + def check_rate_limit(conn, graphql?: true) do + rate_limit_config = Application.get_env(:block_scout_web, Api.GraphQL) + + if rate_limit_config[:rate_limit_disabled?] do + :ok + else + check_graphql_rate_limit_inner(conn, rate_limit_config) + end + end + + defp check_graphql_rate_limit_inner(conn, rate_limit_config) do + global_limit = rate_limit_config[:global_limit] + limit_by_key = rate_limit_config[:limit_by_key] + time_interval_limit = rate_limit_config[:time_interval_limit] + static_api_key = rate_limit_config[:static_api_key] + limit_by_ip = rate_limit_config[:limit_by_ip] + time_interval_by_ip = rate_limit_config[:time_interval_limit_by_ip] + + ip_string = conn_to_ip_string(conn) + plan = get_plan(conn.query_params) + + cond do + check_api_key(conn) && get_api_key(conn) == static_api_key -> + rate_limit(static_api_key, limit_by_key, time_interval_limit) + + check_api_key(conn) && !is_nil(plan) -> + conn + |> get_api_key() + |> rate_limit(min(plan.max_req_per_second, limit_by_key), time_interval_limit) + + true -> + rate_limit("graphql_#{ip_string}", limit_by_ip, time_interval_by_ip) == :ok && + rate_limit("graphql", global_limit, time_interval_limit) == :ok + end + end + def check_rate_limit(conn) do rate_limit_config = Application.get_env(:block_scout_web, :api_rate_limit) diff --git a/config/runtime.exs b/config/runtime.exs index af13203f3d..56d12677eb 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -107,6 +107,8 @@ config :block_scout_web, :api_rate_limit, api_v2_token_ttl_seconds: ConfigHelper.parse_integer_env_var("API_RATE_LIMIT_UI_V2_TOKEN_TTL_IN_SECONDS", 18000), eth_json_rpc_max_batch_size: ConfigHelper.parse_integer_env_var("ETH_JSON_RPC_MAX_BATCH_SIZE", 5) +default_graphql_rate_limit = 10 + config :block_scout_web, Api.GraphQL, default_transaction_hash: System.get_env( @@ -116,7 +118,14 @@ config :block_scout_web, Api.GraphQL, enabled: ConfigHelper.parse_bool_env_var("API_GRAPHQL_ENABLED", "true"), token_limit: ConfigHelper.parse_integer_env_var("API_GRAPHQL_TOKEN_LIMIT", 1000), # Needs to be 215 to support the schema introspection for graphiql - max_complexity: ConfigHelper.parse_integer_env_var("API_GRAPHQL_MAX_COMPLEXITY", 215) + max_complexity: ConfigHelper.parse_integer_env_var("API_GRAPHQL_MAX_COMPLEXITY", 215), + rate_limit_disabled?: ConfigHelper.parse_bool_env_var("API_GRAPHQL_RATE_LIMIT_DISABLED"), + global_limit: ConfigHelper.parse_integer_env_var("API_GRAPHQL_RATE_LIMIT", default_graphql_rate_limit), + limit_by_key: ConfigHelper.parse_integer_env_var("API_GRAPHQL_RATE_LIMIT_BY_KEY", default_graphql_rate_limit), + time_interval_limit: ConfigHelper.parse_time_env_var("API_GRAPHQL_RATE_LIMIT_TIME_INTERVAL", "1s"), + limit_by_ip: ConfigHelper.parse_integer_env_var("API_GRAPHQL_RATE_LIMIT_BY_IP", 500), + time_interval_limit_by_ip: ConfigHelper.parse_time_env_var("API_GRAPHQL_RATE_LIMIT_BY_IP_TIME_INTERVAL", "5m"), + static_api_key: System.get_env("API_GRAPHQL_RATE_LIMIT_STATIC_API_KEY") # Configures History price_chart_config = diff --git a/docker-compose/envs/common-blockscout.env b/docker-compose/envs/common-blockscout.env index da11719d8c..4f1af4df7e 100644 --- a/docker-compose/envs/common-blockscout.env +++ b/docker-compose/envs/common-blockscout.env @@ -67,7 +67,6 @@ HEART_BEAT_TIMEOUT=30 # BLOCKSCOUT_VERSION= RELEASE_LINK= BLOCK_TRANSFORMER=base -# GRAPHIQL_TRANSACTION= # BLOCK_RANGES= # FIRST_BLOCK= # LAST_BLOCK= @@ -126,6 +125,17 @@ API_RATE_LIMIT_WHITELISTED_IPS= API_RATE_LIMIT_STATIC_API_KEY= API_RATE_LIMIT_UI_V2_WITH_TOKEN=5 API_RATE_LIMIT_BY_IP=3000 +# API_GRAPHQL_ENABLED= +# API_GRAPHQL_MAX_COMPLEXITY= +# API_GRAPHQL_TOKEN_LIMIT= +# API_GRAPHQL_DEFAULT_TRANSACTION_HASH= +# API_GRAPHQL_RATE_LIMIT_DISABLED= +# API_GRAPHQL_RATE_LIMIT= +# API_GRAPHQL_RATE_LIMIT_BY_KEY= +# API_GRAPHQL_RATE_LIMIT_TIME_INTERVAL= +# API_GRAPHQL_RATE_LIMIT_BY_IP= +# API_GRAPHQL_RATE_LIMIT_BY_IP_TIME_INTERVAL= +# API_GRAPHQL_RATE_LIMIT_STATIC_API_KEY= DISABLE_INDEXER=false DISABLE_REALTIME_INDEXER=false DISABLE_CATCHUP_INDEXER=false