From d9304c03f8573aebd4286c86f6a461515ef2d578 Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Fri, 2 Oct 2020 14:26:04 +0300 Subject: [PATCH] Handle errors for non-archive node --- CHANGELOG.md | 1 + .../chain/address/coin_balance_daily.ex | 4 +- apps/explorer/lib/explorer/chain/import.ex | 3 ++ apps/indexer/config/dev/geth.exs | 2 +- apps/indexer/config/prod/geth.exs | 2 +- .../lib/indexer/fetcher/coin_balance.ex | 41 ++++++++++-------- .../indexer/fetcher/internal_transaction.ex | 19 +++++++-- .../transform/address_coin_balances_daily.ex | 42 ++++++++++++------- 8 files changed, 75 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 735157945b..b0d3ec9901 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ - [#3256](https://github.com/poanetwork/blockscout/pull/3256) - Fix for invisible validator address at block page and wrong alert text color at xDai ### Chore +- [#3327](https://github.com/poanetwork/blockscout/pull/3327) - Handle various indexer fetchers errors in setup with non-archive node - [#3325](https://github.com/poanetwork/blockscout/pull/3325) - Dark theme improvements - [#3316](https://github.com/poanetwork/blockscout/pull/3316), [#3317](https://github.com/poanetwork/blockscout/pull/3317) - xDai smile logo - [#3315](https://github.com/poanetwork/blockscout/pull/3315) - Environment variable to disable Bridge market cap updater diff --git a/apps/explorer/lib/explorer/chain/address/coin_balance_daily.ex b/apps/explorer/lib/explorer/chain/address/coin_balance_daily.ex index ca90840771..f2c677963a 100644 --- a/apps/explorer/lib/explorer/chain/address/coin_balance_daily.ex +++ b/apps/explorer/lib/explorer/chain/address/coin_balance_daily.ex @@ -64,7 +64,9 @@ defmodule Explorer.Chain.Address.CoinBalanceDaily do ) end - def changeset(%__MODULE__{} = balance, params) do + def changeset(_, params) when is_nil(params), do: :ignore + + def changeset(%__MODULE__{} = balance, params) when not is_nil(params) do balance |> cast(params, @allowed_fields) |> validate_required(@required_fields) diff --git a/apps/explorer/lib/explorer/chain/import.ex b/apps/explorer/lib/explorer/chain/import.ex index cb57baea9c..bdac8cc2c8 100644 --- a/apps/explorer/lib/explorer/chain/import.ex +++ b/apps/explorer/lib/explorer/chain/import.ex @@ -168,6 +168,9 @@ defmodule Explorer.Chain.Import do %Changeset{valid?: true}, {:error, _} = error -> error + + :ignore, error -> + {:error, error} end) |> case do {:ok, changes} -> {:ok, {runner, changes}} diff --git a/apps/indexer/config/dev/geth.exs b/apps/indexer/config/dev/geth.exs index 2132684de1..a3d39c6ea2 100644 --- a/apps/indexer/config/dev/geth.exs +++ b/apps/indexer/config/dev/geth.exs @@ -10,7 +10,7 @@ config :indexer, ), transport_options: [ http: EthereumJSONRPC.HTTP.HTTPoison, - url: System.get_env("ETHEREUM_JSONRPC_HTTP_URL") || "https://mainnet.infura.io/8lTvJTKmHPCHazkneJsY", + url: System.get_env("ETHEREUM_JSONRPC_HTTP_URL"), http_options: [recv_timeout: :timer.minutes(1), timeout: :timer.minutes(1), hackney: [pool: :ethereum_jsonrpc]] ], variant: EthereumJSONRPC.Geth diff --git a/apps/indexer/config/prod/geth.exs b/apps/indexer/config/prod/geth.exs index f70b0057f5..85d8563ce9 100644 --- a/apps/indexer/config/prod/geth.exs +++ b/apps/indexer/config/prod/geth.exs @@ -10,7 +10,7 @@ config :indexer, ), transport_options: [ http: EthereumJSONRPC.HTTP.HTTPoison, - url: System.get_env("ETHEREUM_JSONRPC_HTTP_URL") || "https://mainnet.infura.io/8lTvJTKmHPCHazkneJsY", + url: System.get_env("ETHEREUM_JSONRPC_HTTP_URL"), http_options: [recv_timeout: :timer.minutes(10), timeout: :timer.minutes(10), hackney: [pool: :ethereum_jsonrpc]] ], variant: EthereumJSONRPC.Geth diff --git a/apps/indexer/lib/indexer/fetcher/coin_balance.ex b/apps/indexer/lib/indexer/fetcher/coin_balance.ex index 5212729c2e..3618136f49 100644 --- a/apps/indexer/lib/indexer/fetcher/coin_balance.ex +++ b/apps/indexer/lib/indexer/fetcher/coin_balance.ex @@ -151,25 +151,32 @@ defmodule Indexer.Fetcher.CoinBalance do block_timestamp_map = Enum.reduce(block_numbers, %{}, fn block_number, map -> - {:ok, %Blocks{blocks_params: [%{timestamp: timestamp}]}} = - EthereumJSONRPC.fetch_blocks_by_range(block_number..block_number, json_rpc_named_arguments) - - day = DateTime.to_date(timestamp) - Map.put(map, "#{block_number}", day) + case EthereumJSONRPC.fetch_blocks_by_range(block_number..block_number, json_rpc_named_arguments) do + {:ok, %Blocks{blocks_params: [%{timestamp: timestamp}]}} -> + day = DateTime.to_date(timestamp) + Map.put(map, "#{block_number}", day) + + _ -> + %{} + end end) importable_balances_daily_params = params_list |> Enum.map(fn balance_param -> - day = Map.get(block_timestamp_map, "#{balance_param.block_number}") - - incoming_balance_daily_param = %{ - address_hash: balance_param.address_hash, - day: day, - value: balance_param.value - } - - incoming_balance_daily_param + if Map.has_key?(block_timestamp_map, "#{balance_param.block_number}") do + day = Map.get(block_timestamp_map, "#{balance_param.block_number}") + + incoming_balance_daily_param = %{ + address_hash: balance_param.address_hash, + day: day, + value: balance_param.value + } + + incoming_balance_daily_param + else + nil + end end) addresses_params = balances_params_to_address_params(importable_balances_params) @@ -228,9 +235,9 @@ defmodule Indexer.Fetcher.CoinBalance do end defp run_fetched_balances(%FetchedBalances{errors: errors} = fetched_balances, _) do - {:ok, imported} = import_fetched_balances(fetched_balances) - - Accounts.drop(imported[:addresses]) + with {:ok, imported} <- import_fetched_balances(fetched_balances) do + Accounts.drop(imported[:addresses]) + end retry(errors) end diff --git a/apps/indexer/lib/indexer/fetcher/internal_transaction.ex b/apps/indexer/lib/indexer/fetcher/internal_transaction.ex index 4ed5a85888..0fa0dbba65 100644 --- a/apps/indexer/lib/indexer/fetcher/internal_transaction.ex +++ b/apps/indexer/lib/indexer/fetcher/internal_transaction.ex @@ -109,7 +109,12 @@ defmodule Indexer.Fetcher.InternalTransaction do EthereumJSONRPC.fetch_block_internal_transactions(unique_numbers, json_rpc_named_arguments) _ -> - fetch_block_internal_transactions_by_transactions(unique_numbers, json_rpc_named_arguments) + try do + fetch_block_internal_transactions_by_transactions(unique_numbers, json_rpc_named_arguments) + rescue + error -> + {:error, error} + end end |> case do {:ok, internal_transactions_params} -> @@ -156,8 +161,16 @@ defmodule Indexer.Fetcher.InternalTransaction do |> Chain.get_transactions_of_block_number() |> Enum.map(¶ms(&1)) |> case do - [] -> {:ok, []} - transactions -> EthereumJSONRPC.fetch_internal_transactions(transactions, json_rpc_named_arguments) + [] -> + {:ok, []} + + transactions -> + try do + EthereumJSONRPC.fetch_internal_transactions(transactions, json_rpc_named_arguments) + catch + :exit, error -> + {:error, error} + end end |> case do {:ok, internal_transactions} -> {:ok, internal_transactions ++ acc_list} diff --git a/apps/indexer/lib/indexer/transform/address_coin_balances_daily.ex b/apps/indexer/lib/indexer/transform/address_coin_balances_daily.ex index 8382570ebc..a8f2283d5d 100644 --- a/apps/indexer/lib/indexer/transform/address_coin_balances_daily.ex +++ b/apps/indexer/lib/indexer/transform/address_coin_balances_daily.ex @@ -64,19 +64,26 @@ defmodule Indexer.Transform.AddressCoinBalancesDaily do block_timestamp_map = Enum.reduce(block_numbers, %{}, fn block_number, map -> - {:ok, %Blocks{blocks_params: [%{timestamp: timestamp}]}} = - EthereumJSONRPC.fetch_blocks_by_range(block_number..block_number, json_rpc_named_arguments) - - day = DateTime.to_date(timestamp) - Map.put(map, "#{block_number}", day) + case EthereumJSONRPC.fetch_blocks_by_range(block_number..block_number, json_rpc_named_arguments) do + {:ok, %Blocks{blocks_params: [%{timestamp: timestamp}]}} -> + day = DateTime.to_date(timestamp) + Map.put(map, "#{block_number}", day) + + _ -> + map + end end) logs_params |> Enum.into(acc, fn %{address_hash: address_hash, block_number: block_number} when is_binary(address_hash) and is_integer(block_number) -> - day = Map.get(block_timestamp_map, "#{block_number}") - %{address_hash: address_hash, day: day} + if Map.has_key?(block_timestamp_map, "#{block_number}") do + day = Map.get(block_timestamp_map, "#{block_number}") + %{address_hash: address_hash, day: day} + else + nil + end %{type: "pending"} -> nil @@ -137,19 +144,22 @@ defmodule Indexer.Transform.AddressCoinBalancesDaily do # a transaction MUST have a `from_address_hash` json_rpc_named_arguments = Application.get_env(:explorer, :json_rpc_named_arguments) - {:ok, %Blocks{blocks_params: [%{timestamp: block_timestamp}]}} = - EthereumJSONRPC.fetch_blocks_by_range(block_number..block_number, json_rpc_named_arguments) + case EthereumJSONRPC.fetch_blocks_by_range(block_number..block_number, json_rpc_named_arguments) do + {:ok, %Blocks{blocks_params: [%{timestamp: block_timestamp}]}} -> + day = DateTime.to_date(block_timestamp) + acc = MapSet.put(initial, %{address_hash: from_address_hash, day: day}) - day = DateTime.to_date(block_timestamp) - acc = MapSet.put(initial, %{address_hash: from_address_hash, day: day}) + # `to_address_hash` is optional + case transaction_params do + %{to_address_hash: to_address_hash} when is_binary(to_address_hash) -> + MapSet.put(acc, %{address_hash: to_address_hash, day: day}) - # `to_address_hash` is optional - case transaction_params do - %{to_address_hash: to_address_hash} when is_binary(to_address_hash) -> - MapSet.put(acc, %{address_hash: to_address_hash, day: day}) + _ -> + acc + end _ -> - acc + initial end end end