From 718e63d5f7e85c44fcfbf9d28ec2ad1cbaeee9ed Mon Sep 17 00:00:00 2001 From: nikitosing <32202610+nikitosing@users.noreply.github.com> Date: Fri, 29 Nov 2024 17:06:53 +0300 Subject: [PATCH] feat: Add gzip encoding option (#11292) * feat: Add gzip encoding option * add ETHEREUM_JSONRPC_HTTP_GZIP_ENABLED to common-blockscout.env * Update config/runtime.exs Co-authored-by: Victor Baranov --------- Co-authored-by: Viktor Baranov --- .../lib/ethereum_jsonrpc/http/httpoison.ex | 40 ++++++++++++++++++- config/runtime.exs | 3 +- cspell.json | 7 ++-- docker-compose/envs/common-blockscout.env | 30 +++++++++----- 4 files changed, 64 insertions(+), 16 deletions(-) diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/http/httpoison.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/http/httpoison.ex index d7f5610751..3cc475d3de 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/http/httpoison.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/http/httpoison.ex @@ -9,9 +9,18 @@ defmodule EthereumJSONRPC.HTTP.HTTPoison do @impl HTTP def json_rpc(url, json, headers, options) when is_binary(url) and is_list(options) do + gzip_enabled? = Application.get_env(:ethereum_jsonrpc, EthereumJSONRPC.HTTP)[:gzip_enabled?] + + headers = + if gzip_enabled? do + [{"Accept-Encoding", "gzip"} | headers] + else + headers + end + case HTTPoison.post(url, json, headers, options) do - {:ok, %HTTPoison.Response{body: body, status_code: status_code}} -> - {:ok, %{body: body, status_code: status_code}} + {:ok, %HTTPoison.Response{body: body, status_code: status_code, headers: headers}} -> + {:ok, %{body: try_unzip(gzip_enabled?, body, headers), status_code: status_code}} {:error, %HTTPoison.Error{reason: reason}} -> {:error, reason} @@ -19,4 +28,31 @@ defmodule EthereumJSONRPC.HTTP.HTTPoison do end def json_rpc(url, _json, _headers, _options) when is_nil(url), do: {:error, "URL is nil"} + + defp try_unzip(true, body, headers) do + gzipped = + Enum.any?( + headers + |> Enum.map(fn {k, v} -> + {String.downcase(k), String.downcase(v)} + end), + fn kv -> + case kv do + {"content-encoding", "gzip"} -> true + {"content-encoding", "x-gzip"} -> true + _ -> false + end + end + ) + + if gzipped do + :zlib.gunzip(body) + else + body + end + end + + defp try_unzip(_gzip_enabled?, body, _headers) do + body + end end diff --git a/config/runtime.exs b/config/runtime.exs index 263bf4b34a..d15ac01c21 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -189,7 +189,8 @@ config :ethereum_jsonrpc, EthereumJSONRPC.HTTP, headers: %{"Content-Type" => "application/json"} |> Map.merge(ConfigHelper.parse_json_env_var("ETHEREUM_JSONRPC_HTTP_HEADERS", "{}")) - |> Map.to_list() + |> Map.to_list(), + gzip_enabled?: ConfigHelper.parse_bool_env_var("ETHEREUM_JSONRPC_HTTP_GZIP_ENABLED", "false") config :ethereum_jsonrpc, EthereumJSONRPC.Geth, block_traceable?: ConfigHelper.parse_bool_env_var("ETHEREUM_JSONRPC_GETH_TRACE_BY_BLOCK"), diff --git a/cspell.json b/cspell.json index 0d89d80635..13e7926260 100644 --- a/cspell.json +++ b/cspell.json @@ -78,8 +78,8 @@ "blockreward", "blockscout", "blockscoutuser", - "bools", "Boneh", + "bools", "bridgedtokenlist", "brotli", "browserconfig", @@ -255,6 +255,7 @@ "grecaptcha", "greymatter", "gtag", + "gzipped", "happygokitty", "haspopup", "Hazkne", @@ -643,12 +644,12 @@ "xakgj", "xbaddress", "xdai", - "Xname", "xffff", "xlevel", "xlink", "xmark", "xmlhttprequest", + "Xname", "xnonsense", "xzzz", "yellowgreen", @@ -671,5 +672,5 @@ "dotenv", "html-eex", "makefile" - ], + ] } diff --git a/docker-compose/envs/common-blockscout.env b/docker-compose/envs/common-blockscout.env index 1782375eb0..594c97a5c4 100644 --- a/docker-compose/envs/common-blockscout.env +++ b/docker-compose/envs/common-blockscout.env @@ -1,30 +1,41 @@ ETHEREUM_JSONRPC_VARIANT=geth ETHEREUM_JSONRPC_HTTP_URL=http://host.docker.internal:8545/ -# ETHEREUM_JSONRPC_FALLBACK_HTTP_URL= DATABASE_URL=postgresql://blockscout:ceWb1MeLBEeOIfk65gU8EjF8@db:5432/blockscout + # DATABASE_EVENT_URL= # DATABASE_QUEUE_TARGET # TEST_DATABASE_URL= # TEST_DATABASE_READ_ONLY_API_URL= + +ETHEREUM_JSONRPC_TRANSPORT=http +ETHEREUM_JSONRPC_DISABLE_ARCHIVE_BALANCES=false +# ETHEREUM_JSONRPC_FALLBACK_HTTP_URL= ETHEREUM_JSONRPC_TRACE_URL=http://host.docker.internal:8545/ # ETHEREUM_JSONRPC_FALLBACK_TRACE_URL= -# ETHEREUM_JSONRPC_FALLBACK_ETH_CALL_URL= # ETHEREUM_JSONRPC_ETH_CALL_URL= -# ETHEREUM_JSONRPC_HTTP_TIMEOUT= -# CHAIN_TYPE= -NETWORK= -SUBNETWORK=Awesome chain -LOGO=/images/blockscout_logo.svg +# ETHEREUM_JSONRPC_FALLBACK_ETH_CALL_URL= # ETHEREUM_JSONRPC_WS_URL= # ETHEREUM_JSONRPC_FALLBACK_WS_URL= # ETHEREUM_JSONRPC_WS_RETRY_INTERVAL= -ETHEREUM_JSONRPC_TRANSPORT=http -ETHEREUM_JSONRPC_DISABLE_ARCHIVE_BALANCES=false # ETHEREUM_JSONRPC_ARCHIVE_BALANCES_WINDOW=200 +# ETHEREUM_JSONRPC_HTTP_TIMEOUT= # ETHEREUM_JSONRPC_HTTP_HEADERS= +# ETHEREUM_JSONRPC_HTTP_GZIP_ENABLED= # ETHEREUM_JSONRPC_WAIT_PER_TIMEOUT= # ETHEREUM_JSONRPC_GETH_TRACE_BY_BLOCK= # ETHEREUM_JSONRPC_GETH_ALLOW_EMPTY_TRACES= +# ETHEREUM_JSONRPC_DEBUG_TRACE_TRANSACTION_TIMEOUT= +# ETHEREUM_JSONRPC_HTTP_URLS= +# ETHEREUM_JSONRPC_FALLBACK_HTTP_URLS= +# ETHEREUM_JSONRPC_TRACE_URLS= +# ETHEREUM_JSONRPC_FALLBACK_TRACE_URLS= +# ETHEREUM_JSONRPC_ETH_CALL_URLS= +# ETHEREUM_JSONRPC_FALLBACK_ETH_CALL_URLS= + +# CHAIN_TYPE= +NETWORK= +SUBNETWORK=Awesome chain +LOGO=/images/blockscout_logo.svg IPC_PATH= NETWORK_PATH=/ BLOCKSCOUT_HOST= @@ -175,7 +186,6 @@ INDEXER_DISABLE_INTERNAL_TRANSACTIONS_FETCHER=false # INDEXER_EMPTY_BLOCKS_SANITIZER_INTERVAL= # INDEXER_INTERNAL_TRANSACTIONS_BATCH_SIZE= # INDEXER_INTERNAL_TRANSACTIONS_CONCURRENCY= -# ETHEREUM_JSONRPC_DEBUG_TRACE_TRANSACTION_TIMEOUT= # INDEXER_BLOCK_REWARD_BATCH_SIZE= # INDEXER_BLOCK_REWARD_CONCURRENCY= # INDEXER_TOKEN_INSTANCE_USE_BASE_URI_RETRY=