From 4049b036cc36ef2a93da42237d8835bc9e37658f Mon Sep 17 00:00:00 2001 From: Qwerty5Uiop <105209995+Qwerty5Uiop@users.noreply.github.com> Date: Fri, 3 May 2024 19:08:43 +0400 Subject: [PATCH] feat: Omit balanceOf requests for tokens that doesn't support it (#10018) * feat: Omit balanceOf requests for tokens that doesn't support it * Missing balanceOf token refactoring * Fix failed token balance error matching --- .../chain/address/current_token_balance.ex | 10 ++ .../explorer/chain/address/token_balance.ex | 25 ++++- .../runner/address/current_token_balances.ex | 10 +- .../import/runner/address/token_balances.ex | 27 ++++- .../utility/missing_balance_of_token.ex | 101 ++++++++++++++++++ ...64431_create_missing_balance_of_tokens.exs | 12 +++ .../runner/address/token_balances_test.exs | 41 +++++++ apps/explorer/test/support/factory.ex | 9 +- .../lib/indexer/fetcher/token_balance.ex | 25 ++++- apps/indexer/lib/indexer/token_balances.ex | 2 +- .../indexer/fetcher/token_balance_test.exs | 63 ++++++++++- cspell.json | 1 + 12 files changed, 316 insertions(+), 10 deletions(-) create mode 100644 apps/explorer/lib/explorer/utility/missing_balance_of_token.ex create mode 100644 apps/explorer/priv/repo/migrations/20240502064431_create_missing_balance_of_tokens.exs diff --git a/apps/explorer/lib/explorer/chain/address/current_token_balance.ex b/apps/explorer/lib/explorer/chain/address/current_token_balance.ex index 7e8db74df6..b248ee4479 100644 --- a/apps/explorer/lib/explorer/chain/address/current_token_balance.ex +++ b/apps/explorer/lib/explorer/chain/address/current_token_balance.ex @@ -14,6 +14,7 @@ defmodule Explorer.Chain.Address.CurrentTokenBalance do alias Explorer.{Chain, PagingOptions, Repo} alias Explorer.Chain.{Address, Block, CurrencyHelper, Hash, Token} + alias Explorer.Chain.Address.TokenBalance @default_paging_options %PagingOptions{page_size: 50} @@ -322,6 +323,15 @@ defmodule Explorer.Chain.Address.CurrentTokenBalance do Repo.one!(query, timeout: :infinity) end + @doc """ + Deletes all CurrentTokenBalances with given `token_contract_address_hash` and below the given `block_number`. + Used for cases when token doesn't implement balanceOf function + """ + @spec delete_placeholders_below(Hash.Address.t(), Block.block_number()) :: {non_neg_integer(), nil | [term()]} + def delete_placeholders_below(token_contract_address_hash, block_number) do + TokenBalance.delete_token_balance_placeholders_below(__MODULE__, token_contract_address_hash, block_number) + end + @doc """ Converts CurrentTokenBalances to CSV format. Used in `BlockScoutWeb.API.V2.CSVExportController.export_token_holders/2` """ diff --git a/apps/explorer/lib/explorer/chain/address/token_balance.ex b/apps/explorer/lib/explorer/chain/address/token_balance.ex index e50462b1e9..e0aef47563 100644 --- a/apps/explorer/lib/explorer/chain/address/token_balance.ex +++ b/apps/explorer/lib/explorer/chain/address/token_balance.ex @@ -11,7 +11,7 @@ defmodule Explorer.Chain.Address.TokenBalance do import Explorer.Chain.SmartContract, only: [burn_address_hash_string: 0] - alias Explorer.Chain + alias Explorer.{Chain, Repo} alias Explorer.Chain.Address.TokenBalance alias Explorer.Chain.Cache.BackgroundMigrations alias Explorer.Chain.{Address, Block, Hash, Token} @@ -119,4 +119,27 @@ defmodule Explorer.Chain.Address.TokenBalance do order_by: [desc: :block_number] ) end + + @doc """ + Deletes all token balances with given `token_contract_address_hash` and below the given `block_number`. + Used for cases when token doesn't implement `balanceOf` function + """ + @spec delete_placeholders_below(Hash.Address.t(), Block.block_number()) :: {non_neg_integer(), nil | [term()]} + def delete_placeholders_below(token_contract_address_hash, block_number) do + delete_token_balance_placeholders_below(__MODULE__, token_contract_address_hash, block_number) + end + + @doc """ + Deletes all token balances or current token balances with given `token_contract_address_hash` and below the given `block_number`. + Used for cases when token doesn't implement `balanceOf` function + """ + @spec delete_token_balance_placeholders_below(atom(), Hash.Address.t(), Block.block_number()) :: + {non_neg_integer(), nil | [term()]} + def delete_token_balance_placeholders_below(module, token_contract_address_hash, block_number) do + module + |> where([tb], tb.token_contract_address_hash == ^token_contract_address_hash) + |> where([tb], tb.block_number <= ^block_number) + |> where([tb], is_nil(tb.value_fetched_at) or is_nil(tb.value)) + |> Repo.delete_all() + end end diff --git a/apps/explorer/lib/explorer/chain/import/runner/address/current_token_balances.ex b/apps/explorer/lib/explorer/chain/import/runner/address/current_token_balances.ex index 1b47a51ab6..1ea20e0c3d 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/address/current_token_balances.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/address/current_token_balances.ex @@ -10,7 +10,7 @@ defmodule Explorer.Chain.Import.Runner.Address.CurrentTokenBalances do alias Ecto.{Changeset, Multi, Repo} alias Explorer.Chain.Address.CurrentTokenBalance alias Explorer.Chain.{Hash, Import} - alias Explorer.Chain.Import.Runner.Tokens + alias Explorer.Chain.Import.Runner.{Address.TokenBalances, Tokens} alias Explorer.Prometheus.Instrumenter @behaviour Import.Runner @@ -108,6 +108,14 @@ defmodule Explorer.Chain.Import.Runner.Address.CurrentTokenBalances do |> Map.put(:timestamps, timestamps) multi + |> Multi.run(:filter_placeholders, fn _, _ -> + Instrumenter.block_import_stage_runner( + fn -> TokenBalances.filter_placeholders(changes_list) end, + :block_following, + :current_token_balances, + :filter_placeholders + ) + end) |> Multi.run(:address_current_token_balances, fn repo, _ -> Instrumenter.block_import_stage_runner( fn -> insert(repo, changes_list, insert_options) end, diff --git a/apps/explorer/lib/explorer/chain/import/runner/address/token_balances.ex b/apps/explorer/lib/explorer/chain/import/runner/address/token_balances.ex index 61153b7b98..e4e20d08fa 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/address/token_balances.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/address/token_balances.ex @@ -11,6 +11,7 @@ defmodule Explorer.Chain.Import.Runner.Address.TokenBalances do alias Explorer.Chain.Address.TokenBalance alias Explorer.Chain.Import alias Explorer.Prometheus.Instrumenter + alias Explorer.Utility.MissingBalanceOfToken @behaviour Import.Runner @@ -42,9 +43,18 @@ defmodule Explorer.Chain.Import.Runner.Address.TokenBalances do |> Map.put_new(:timeout, @timeout) |> Map.put(:timestamps, timestamps) - Multi.run(multi, :address_token_balances, fn repo, _ -> + multi + |> Multi.run(:filter_placeholders, fn _, _ -> Instrumenter.block_import_stage_runner( - fn -> insert(repo, changes_list, insert_options) end, + fn -> filter_placeholders(changes_list) end, + :block_referencing, + :token_balances, + :filter_placeholders + ) + end) + |> Multi.run(:address_token_balances, fn repo, %{filter_placeholders: filtered_changes_list} -> + Instrumenter.block_import_stage_runner( + fn -> insert(repo, filtered_changes_list, insert_options) end, :block_referencing, :token_balances, :address_token_balances @@ -55,6 +65,19 @@ defmodule Explorer.Chain.Import.Runner.Address.TokenBalances do @impl Import.Runner def timeout, do: @timeout + @doc """ + Filters out changes with empty `value` or `value_fetched_at` for tokens that doesn't implement `balanceOf` function. + """ + @spec filter_placeholders([map()]) :: {:ok, [map()]} + def filter_placeholders(changes_list) do + {placeholders, filled_balances} = + Enum.split_with(changes_list, fn balance_params -> + is_nil(Map.get(balance_params, :value_fetched_at)) or is_nil(Map.get(balance_params, :value)) + end) + + {:ok, filled_balances ++ MissingBalanceOfToken.filter_token_balances_params(placeholders)} + end + @spec insert(Repo.t(), [map()], %{ optional(:on_conflict) => Import.Runner.on_conflict(), required(:timeout) => timeout(), diff --git a/apps/explorer/lib/explorer/utility/missing_balance_of_token.ex b/apps/explorer/lib/explorer/utility/missing_balance_of_token.ex new file mode 100644 index 0000000000..671de226ba --- /dev/null +++ b/apps/explorer/lib/explorer/utility/missing_balance_of_token.ex @@ -0,0 +1,101 @@ +defmodule Explorer.Utility.MissingBalanceOfToken do + @moduledoc """ + Module is responsible for keeping address hashes of tokens that does not support the balanceOf function + and the maximum block number for which this function call returned an error. + """ + + use Explorer.Schema + + alias Explorer.Chain.{Hash, Token} + alias Explorer.Repo + + @primary_key false + typed_schema "missing_balance_of_tokens" do + field(:block_number, :integer) + + belongs_to( + :token, + Token, + foreign_key: :token_contract_address_hash, + references: :contract_address_hash, + primary_key: true, + type: Hash.Address, + null: false + ) + + timestamps() + end + + @doc false + def changeset(missing_balance_of_token \\ %__MODULE__{}, params) do + cast(missing_balance_of_token, params, [:token_contract_address_hash, :block_number]) + end + + @doc """ + Returns all records by provided token contract address hashes + """ + @spec get_by_hashes([Hash.Address.t()]) :: [%__MODULE__{}] + def get_by_hashes(token_contract_address_hashes) do + __MODULE__ + |> where([mbot], mbot.token_contract_address_hash in ^token_contract_address_hashes) + |> Repo.all() + end + + @doc """ + Filters provided token balances params by presence of record with the same `token_contract_address_hash` + and above or equal `block_number` in `missing_balance_of_tokens`. + """ + @spec filter_token_balances_params([map()]) :: [map()] + def filter_token_balances_params(params) do + missing_balance_of_tokens_map = + params + |> Enum.map(& &1.token_contract_address_hash) + |> get_by_hashes() + |> Enum.map(&{to_string(&1.token_contract_address_hash), &1.block_number}) + |> Map.new() + + Enum.filter(params, fn %{token_contract_address_hash: token_contract_address_hash, block_number: block_number} -> + case missing_balance_of_tokens_map[to_string(token_contract_address_hash)] do + nil -> true + missing_balance_of_block_number -> block_number > missing_balance_of_block_number + end + end) + end + + @doc """ + Inserts new `missing_balance_of_tokens` records by provided params (except for `ERC-404` token type) + """ + @spec insert_from_params([map()]) :: {non_neg_integer(), nil | [term()]} + def insert_from_params(token_balance_params) do + now = DateTime.utc_now() + + params = + token_balance_params + |> Enum.reject(&(&1.token_type == "ERC-404")) + |> Enum.group_by(& &1.token_contract_address_hash, & &1.block_number) + |> Enum.map(fn {token_contract_address_hash, block_numbers} -> + {:ok, token_contract_address_hash_casted} = Hash.Address.cast(token_contract_address_hash) + + %{ + token_contract_address_hash: token_contract_address_hash_casted, + block_number: Enum.max(block_numbers), + inserted_at: now, + updated_at: now + } + end) + + Repo.insert_all(__MODULE__, params, on_conflict: on_conflict(), conflict_target: :token_contract_address_hash) + end + + defp on_conflict do + from( + mbot in __MODULE__, + update: [ + set: [ + block_number: fragment("GREATEST(EXCLUDED.block_number, ?)", mbot.block_number), + updated_at: fragment("EXCLUDED.updated_at") + ] + ] + ) + end +end diff --git a/apps/explorer/priv/repo/migrations/20240502064431_create_missing_balance_of_tokens.exs b/apps/explorer/priv/repo/migrations/20240502064431_create_missing_balance_of_tokens.exs new file mode 100644 index 0000000000..a75513c027 --- /dev/null +++ b/apps/explorer/priv/repo/migrations/20240502064431_create_missing_balance_of_tokens.exs @@ -0,0 +1,12 @@ +defmodule Explorer.Repo.Migrations.CreateMissingBalanceOfTokens do + use Ecto.Migration + + def change do + create table(:missing_balance_of_tokens, primary_key: false) do + add(:token_contract_address_hash, :bytea, primary_key: true) + add(:block_number, :bigint) + + timestamps() + end + end +end diff --git a/apps/explorer/test/explorer/chain/import/runner/address/token_balances_test.exs b/apps/explorer/test/explorer/chain/import/runner/address/token_balances_test.exs index 40034eb841..f391aaeb80 100644 --- a/apps/explorer/test/explorer/chain/import/runner/address/token_balances_test.exs +++ b/apps/explorer/test/explorer/chain/import/runner/address/token_balances_test.exs @@ -219,6 +219,47 @@ defmodule Explorer.Chain.Import.Runner.Address.TokenBalancesTest do }} = run_changes(second_changes, options) end + test "filters out changes with tokens that doesn't implement balanceOf function" do + address = insert(:address) + token = insert(:token) + + options = %{ + timeout: :infinity, + timestamps: %{inserted_at: DateTime.utc_now(), updated_at: DateTime.utc_now()} + } + + block_number = 1 + next_block_number = block_number + 1 + token_contract_address_hash = token.contract_address_hash + + insert(:missing_balance_of_token, + token_contract_address_hash: token_contract_address_hash, + block_number: block_number + ) + + address_hash = address.hash + + changes_list = [ + %{ + address_hash: address_hash, + block_number: block_number, + token_contract_address_hash: token_contract_address_hash, + token_id: 11, + token_type: "ERC-721" + }, + %{ + address_hash: address_hash, + block_number: next_block_number, + token_contract_address_hash: token_contract_address_hash, + token_id: 12, + token_type: "ERC-721" + } + ] + + assert {:ok, %{address_token_balances: [%{block_number: ^next_block_number}]}} = + run_changes_list(changes_list, options) + end + defp run_changes(changes, options) when is_map(changes) do run_changes_list([changes], options) end diff --git a/apps/explorer/test/support/factory.ex b/apps/explorer/test/support/factory.ex index 099a2b47c3..90e5181154 100644 --- a/apps/explorer/test/support/factory.ex +++ b/apps/explorer/test/support/factory.ex @@ -55,7 +55,7 @@ defmodule Explorer.Factory do alias Explorer.Market.MarketHistory alias Explorer.Repo - alias Explorer.Utility.MissingBlockRange + alias Explorer.Utility.{MissingBalanceOfToken, MissingBlockRange} alias Ueberauth.Strategy.Auth0 alias Ueberauth.Auth.Info @@ -1090,6 +1090,13 @@ defmodule Explorer.Factory do } end + def missing_balance_of_token_factory do + %MissingBalanceOfToken{ + token_contract_address_hash: insert(:token).contract_address_hash, + block_number: block_number() + } + end + def withdrawal_factory do block = build(:block) address = build(:address) diff --git a/apps/indexer/lib/indexer/fetcher/token_balance.ex b/apps/indexer/lib/indexer/fetcher/token_balance.ex index 29b3bad043..bfabded15a 100644 --- a/apps/indexer/lib/indexer/fetcher/token_balance.ex +++ b/apps/indexer/lib/indexer/fetcher/token_balance.ex @@ -19,7 +19,9 @@ defmodule Indexer.Fetcher.TokenBalance do require Logger alias Explorer.Chain + alias Explorer.Chain.Address.{CurrentTokenBalance, TokenBalance} alias Explorer.Chain.Hash + alias Explorer.Utility.MissingBalanceOfToken alias Indexer.{BufferedTask, TokenBalances, Tracer} alias Indexer.Fetcher.TokenBalance.Supervisor, as: TokenBalanceSupervisor @@ -97,6 +99,7 @@ defmodule Indexer.Fetcher.TokenBalance do result = entries |> Enum.map(&format_params/1) + |> MissingBalanceOfToken.filter_token_balances_params() |> increase_retries_count() |> fetch_from_blockchain() |> import_token_balances() @@ -130,7 +133,10 @@ defmodule Indexer.Fetcher.TokenBalance do if Enum.empty?(failed_token_balances) do {:halt, all_token_balances} else - failed_token_balances = increase_retries_count(failed_token_balances) + failed_token_balances = + failed_token_balances + |> handle_failed_balances() + |> increase_retries_count() token_balances_updated_retries_count = all_token_balances @@ -143,6 +149,23 @@ defmodule Indexer.Fetcher.TokenBalance do fetched_token_balances end + defp handle_failed_balances(failed_token_balances) do + {missing_balance_of_balances, other_failed_balances} = + Enum.split_with(failed_token_balances, fn + %{error: error} when is_binary(error) -> error =~ "execution reverted" + _ -> false + end) + + MissingBalanceOfToken.insert_from_params(missing_balance_of_balances) + + Enum.each(missing_balance_of_balances, fn balance -> + TokenBalance.delete_placeholders_below(balance.token_contract_address_hash, balance.block_number) + CurrentTokenBalance.delete_placeholders_below(balance.token_contract_address_hash, balance.block_number) + end) + + other_failed_balances + end + defp increase_retries_count(params_list) do params_list |> Enum.map(&Map.put(&1, :retries_count, &1.retries_count + 1)) diff --git a/apps/indexer/lib/indexer/token_balances.ex b/apps/indexer/lib/indexer/token_balances.ex index 066cddbac2..da6ecb56e3 100644 --- a/apps/indexer/lib/indexer/token_balances.ex +++ b/apps/indexer/lib/indexer/token_balances.ex @@ -41,7 +41,7 @@ defmodule Indexer.TokenBalances do * `token_type` - type of the token that balance belongs to * `token_id` - token id for ERC-1155/ERC-404 tokens """ - def fetch_token_balances_from_blockchain([]), do: {:ok, []} + def fetch_token_balances_from_blockchain([]), do: {:ok, %{fetched_token_balances: [], failed_token_balances: []}} @decorate span(tracer: Tracer) def fetch_token_balances_from_blockchain(token_balances) do diff --git a/apps/indexer/test/indexer/fetcher/token_balance_test.exs b/apps/indexer/test/indexer/fetcher/token_balance_test.exs index 3da5675d52..5d33543b75 100644 --- a/apps/indexer/test/indexer/fetcher/token_balance_test.exs +++ b/apps/indexer/test/indexer/fetcher/token_balance_test.exs @@ -5,6 +5,8 @@ defmodule Indexer.Fetcher.TokenBalanceTest do import Mox alias Explorer.Chain.{Address, Hash} + alias Explorer.Repo + alias Explorer.Utility.MissingBalanceOfToken alias Indexer.Fetcher.TokenBalance @moduletag :capture_log @@ -62,7 +64,7 @@ defmodule Indexer.Fetcher.TokenBalanceTest do nil ) == :ok - token_balance_updated = Explorer.Repo.get_by(Address.TokenBalance, address_hash: address_hash) + token_balance_updated = Repo.get_by(Address.TokenBalance, address_hash: address_hash) assert token_balance_updated.value == Decimal.new(1_000_000_000_000_000_000_000_000) assert token_balance_updated.value_fetched_at != nil @@ -110,7 +112,7 @@ defmodule Indexer.Fetcher.TokenBalanceTest do nil ) == :ok - token_balance_updated = Explorer.Repo.get_by(Address.TokenBalance, address_hash: address_hash) + token_balance_updated = Repo.get_by(Address.TokenBalance, address_hash: address_hash) assert token_balance_updated.value == Decimal.new(1_000_000_000_000_000_000_000_000) assert token_balance_updated.value_fetched_at != nil @@ -180,7 +182,62 @@ defmodule Indexer.Fetcher.TokenBalanceTest do assert 1 = from(tb in Address.TokenBalance, where: tb.address_hash == ^address_hash) - |> Explorer.Repo.aggregate(:count, :id) + |> Repo.aggregate(:count, :id) + end + + test "filters out params with tokens that doesn't implement balanceOf function" do + address = insert(:address) + missing_balance_of_token = insert(:missing_balance_of_token) + + assert TokenBalance.run( + [ + {address.hash.bytes, missing_balance_of_token.token_contract_address_hash.bytes, + missing_balance_of_token.block_number, "ERC-20", nil, 0} + ], + nil + ) == :ok + + assert Repo.all(Address.TokenBalance) == [] + end + + test "in case of error deletes token balance placeholders below the given number and inserts new missing balanceOf tokens" do + address = insert(:address) + %{contract_address_hash: token_contract_address_hash} = insert(:token) + + insert(:token_balance, + token_contract_address_hash: token_contract_address_hash, + address: address, + block_number: 0, + value_fetched_at: nil, + value: nil + ) + + expect( + EthereumJSONRPC.Mox, + :json_rpc, + fn [%{id: id, method: "eth_call", params: [%{data: _, to: _}, _]}], _options -> + {:ok, + [ + %{ + id: id, + jsonrpc: "2.0", + error: %{code: "-32000", message: "execution reverted"} + } + ]} + end + ) + + assert TokenBalance.run( + [ + {address.hash.bytes, token_contract_address_hash.bytes, 1, "ERC-20", nil, 0} + ], + nil + ) == :ok + + assert %{token_contract_address_hash: ^token_contract_address_hash, block_number: 1} = + Repo.one(MissingBalanceOfToken) + + assert Repo.all(Address.TokenBalance) == [] end end diff --git a/cspell.json b/cspell.json index 5ab24e0e43..065e1e2d80 100644 --- a/cspell.json +++ b/cspell.json @@ -337,6 +337,7 @@ "malihu", "mallowance", "maxlength", + "mbot", "mcap", "mconst", "mdef",