feat: Omit balanceOf requests for tokens that doesn't support it (#10018)

* feat: Omit balanceOf requests for tokens that doesn't support it

* Missing balanceOf token refactoring

* Fix failed token balance error matching
pull/10028/head
Qwerty5Uiop 6 months ago committed by GitHub
parent a7eb86f2be
commit 4049b036cc
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
  1. 10
      apps/explorer/lib/explorer/chain/address/current_token_balance.ex
  2. 25
      apps/explorer/lib/explorer/chain/address/token_balance.ex
  3. 10
      apps/explorer/lib/explorer/chain/import/runner/address/current_token_balances.ex
  4. 27
      apps/explorer/lib/explorer/chain/import/runner/address/token_balances.ex
  5. 101
      apps/explorer/lib/explorer/utility/missing_balance_of_token.ex
  6. 12
      apps/explorer/priv/repo/migrations/20240502064431_create_missing_balance_of_tokens.exs
  7. 41
      apps/explorer/test/explorer/chain/import/runner/address/token_balances_test.exs
  8. 9
      apps/explorer/test/support/factory.ex
  9. 25
      apps/indexer/lib/indexer/fetcher/token_balance.ex
  10. 2
      apps/indexer/lib/indexer/token_balances.ex
  11. 63
      apps/indexer/test/indexer/fetcher/token_balance_test.exs
  12. 1
      cspell.json

@ -14,6 +14,7 @@ defmodule Explorer.Chain.Address.CurrentTokenBalance do
alias Explorer.{Chain, PagingOptions, Repo}
alias Explorer.Chain.{Address, Block, CurrencyHelper, Hash, Token}
alias Explorer.Chain.Address.TokenBalance
@default_paging_options %PagingOptions{page_size: 50}
@ -322,6 +323,15 @@ defmodule Explorer.Chain.Address.CurrentTokenBalance do
Repo.one!(query, timeout: :infinity)
end
@doc """
Deletes all CurrentTokenBalances with given `token_contract_address_hash` and below the given `block_number`.
Used for cases when token doesn't implement balanceOf function
"""
@spec delete_placeholders_below(Hash.Address.t(), Block.block_number()) :: {non_neg_integer(), nil | [term()]}
def delete_placeholders_below(token_contract_address_hash, block_number) do
TokenBalance.delete_token_balance_placeholders_below(__MODULE__, token_contract_address_hash, block_number)
end
@doc """
Converts CurrentTokenBalances to CSV format. Used in `BlockScoutWeb.API.V2.CSVExportController.export_token_holders/2`
"""

@ -11,7 +11,7 @@ defmodule Explorer.Chain.Address.TokenBalance do
import Explorer.Chain.SmartContract, only: [burn_address_hash_string: 0]
alias Explorer.Chain
alias Explorer.{Chain, Repo}
alias Explorer.Chain.Address.TokenBalance
alias Explorer.Chain.Cache.BackgroundMigrations
alias Explorer.Chain.{Address, Block, Hash, Token}
@ -119,4 +119,27 @@ defmodule Explorer.Chain.Address.TokenBalance do
order_by: [desc: :block_number]
)
end
@doc """
Deletes all token balances with given `token_contract_address_hash` and below the given `block_number`.
Used for cases when token doesn't implement `balanceOf` function
"""
@spec delete_placeholders_below(Hash.Address.t(), Block.block_number()) :: {non_neg_integer(), nil | [term()]}
def delete_placeholders_below(token_contract_address_hash, block_number) do
delete_token_balance_placeholders_below(__MODULE__, token_contract_address_hash, block_number)
end
@doc """
Deletes all token balances or current token balances with given `token_contract_address_hash` and below the given `block_number`.
Used for cases when token doesn't implement `balanceOf` function
"""
@spec delete_token_balance_placeholders_below(atom(), Hash.Address.t(), Block.block_number()) ::
{non_neg_integer(), nil | [term()]}
def delete_token_balance_placeholders_below(module, token_contract_address_hash, block_number) do
module
|> where([tb], tb.token_contract_address_hash == ^token_contract_address_hash)
|> where([tb], tb.block_number <= ^block_number)
|> where([tb], is_nil(tb.value_fetched_at) or is_nil(tb.value))
|> Repo.delete_all()
end
end

@ -10,7 +10,7 @@ defmodule Explorer.Chain.Import.Runner.Address.CurrentTokenBalances do
alias Ecto.{Changeset, Multi, Repo}
alias Explorer.Chain.Address.CurrentTokenBalance
alias Explorer.Chain.{Hash, Import}
alias Explorer.Chain.Import.Runner.Tokens
alias Explorer.Chain.Import.Runner.{Address.TokenBalances, Tokens}
alias Explorer.Prometheus.Instrumenter
@behaviour Import.Runner
@ -108,6 +108,14 @@ defmodule Explorer.Chain.Import.Runner.Address.CurrentTokenBalances do
|> Map.put(:timestamps, timestamps)
multi
|> Multi.run(:filter_placeholders, fn _, _ ->
Instrumenter.block_import_stage_runner(
fn -> TokenBalances.filter_placeholders(changes_list) end,
:block_following,
:current_token_balances,
:filter_placeholders
)
end)
|> Multi.run(:address_current_token_balances, fn repo, _ ->
Instrumenter.block_import_stage_runner(
fn -> insert(repo, changes_list, insert_options) end,

@ -11,6 +11,7 @@ defmodule Explorer.Chain.Import.Runner.Address.TokenBalances do
alias Explorer.Chain.Address.TokenBalance
alias Explorer.Chain.Import
alias Explorer.Prometheus.Instrumenter
alias Explorer.Utility.MissingBalanceOfToken
@behaviour Import.Runner
@ -42,9 +43,18 @@ defmodule Explorer.Chain.Import.Runner.Address.TokenBalances do
|> Map.put_new(:timeout, @timeout)
|> Map.put(:timestamps, timestamps)
Multi.run(multi, :address_token_balances, fn repo, _ ->
multi
|> Multi.run(:filter_placeholders, fn _, _ ->
Instrumenter.block_import_stage_runner(
fn -> insert(repo, changes_list, insert_options) end,
fn -> filter_placeholders(changes_list) end,
:block_referencing,
:token_balances,
:filter_placeholders
)
end)
|> Multi.run(:address_token_balances, fn repo, %{filter_placeholders: filtered_changes_list} ->
Instrumenter.block_import_stage_runner(
fn -> insert(repo, filtered_changes_list, insert_options) end,
:block_referencing,
:token_balances,
:address_token_balances
@ -55,6 +65,19 @@ defmodule Explorer.Chain.Import.Runner.Address.TokenBalances do
@impl Import.Runner
def timeout, do: @timeout
@doc """
Filters out changes with empty `value` or `value_fetched_at` for tokens that doesn't implement `balanceOf` function.
"""
@spec filter_placeholders([map()]) :: {:ok, [map()]}
def filter_placeholders(changes_list) do
{placeholders, filled_balances} =
Enum.split_with(changes_list, fn balance_params ->
is_nil(Map.get(balance_params, :value_fetched_at)) or is_nil(Map.get(balance_params, :value))
end)
{:ok, filled_balances ++ MissingBalanceOfToken.filter_token_balances_params(placeholders)}
end
@spec insert(Repo.t(), [map()], %{
optional(:on_conflict) => Import.Runner.on_conflict(),
required(:timeout) => timeout(),

@ -0,0 +1,101 @@
defmodule Explorer.Utility.MissingBalanceOfToken do
@moduledoc """
Module is responsible for keeping address hashes of tokens that does not support the balanceOf function
and the maximum block number for which this function call returned an error.
"""
use Explorer.Schema
alias Explorer.Chain.{Hash, Token}
alias Explorer.Repo
@primary_key false
typed_schema "missing_balance_of_tokens" do
field(:block_number, :integer)
belongs_to(
:token,
Token,
foreign_key: :token_contract_address_hash,
references: :contract_address_hash,
primary_key: true,
type: Hash.Address,
null: false
)
timestamps()
end
@doc false
def changeset(missing_balance_of_token \\ %__MODULE__{}, params) do
cast(missing_balance_of_token, params, [:token_contract_address_hash, :block_number])
end
@doc """
Returns all records by provided token contract address hashes
"""
@spec get_by_hashes([Hash.Address.t()]) :: [%__MODULE__{}]
def get_by_hashes(token_contract_address_hashes) do
__MODULE__
|> where([mbot], mbot.token_contract_address_hash in ^token_contract_address_hashes)
|> Repo.all()
end
@doc """
Filters provided token balances params by presence of record with the same `token_contract_address_hash`
and above or equal `block_number` in `missing_balance_of_tokens`.
"""
@spec filter_token_balances_params([map()]) :: [map()]
def filter_token_balances_params(params) do
missing_balance_of_tokens_map =
params
|> Enum.map(& &1.token_contract_address_hash)
|> get_by_hashes()
|> Enum.map(&{to_string(&1.token_contract_address_hash), &1.block_number})
|> Map.new()
Enum.filter(params, fn %{token_contract_address_hash: token_contract_address_hash, block_number: block_number} ->
case missing_balance_of_tokens_map[to_string(token_contract_address_hash)] do
nil -> true
missing_balance_of_block_number -> block_number > missing_balance_of_block_number
end
end)
end
@doc """
Inserts new `missing_balance_of_tokens` records by provided params (except for `ERC-404` token type)
"""
@spec insert_from_params([map()]) :: {non_neg_integer(), nil | [term()]}
def insert_from_params(token_balance_params) do
now = DateTime.utc_now()
params =
token_balance_params
|> Enum.reject(&(&1.token_type == "ERC-404"))
|> Enum.group_by(& &1.token_contract_address_hash, & &1.block_number)
|> Enum.map(fn {token_contract_address_hash, block_numbers} ->
{:ok, token_contract_address_hash_casted} = Hash.Address.cast(token_contract_address_hash)
%{
token_contract_address_hash: token_contract_address_hash_casted,
block_number: Enum.max(block_numbers),
inserted_at: now,
updated_at: now
}
end)
Repo.insert_all(__MODULE__, params, on_conflict: on_conflict(), conflict_target: :token_contract_address_hash)
end
defp on_conflict do
from(
mbot in __MODULE__,
update: [
set: [
block_number: fragment("GREATEST(EXCLUDED.block_number, ?)", mbot.block_number),
updated_at: fragment("EXCLUDED.updated_at")
]
]
)
end
end

@ -0,0 +1,12 @@
defmodule Explorer.Repo.Migrations.CreateMissingBalanceOfTokens do
use Ecto.Migration
def change do
create table(:missing_balance_of_tokens, primary_key: false) do
add(:token_contract_address_hash, :bytea, primary_key: true)
add(:block_number, :bigint)
timestamps()
end
end
end

@ -219,6 +219,47 @@ defmodule Explorer.Chain.Import.Runner.Address.TokenBalancesTest do
}} = run_changes(second_changes, options)
end
test "filters out changes with tokens that doesn't implement balanceOf function" do
address = insert(:address)
token = insert(:token)
options = %{
timeout: :infinity,
timestamps: %{inserted_at: DateTime.utc_now(), updated_at: DateTime.utc_now()}
}
block_number = 1
next_block_number = block_number + 1
token_contract_address_hash = token.contract_address_hash
insert(:missing_balance_of_token,
token_contract_address_hash: token_contract_address_hash,
block_number: block_number
)
address_hash = address.hash
changes_list = [
%{
address_hash: address_hash,
block_number: block_number,
token_contract_address_hash: token_contract_address_hash,
token_id: 11,
token_type: "ERC-721"
},
%{
address_hash: address_hash,
block_number: next_block_number,
token_contract_address_hash: token_contract_address_hash,
token_id: 12,
token_type: "ERC-721"
}
]
assert {:ok, %{address_token_balances: [%{block_number: ^next_block_number}]}} =
run_changes_list(changes_list, options)
end
defp run_changes(changes, options) when is_map(changes) do
run_changes_list([changes], options)
end

@ -55,7 +55,7 @@ defmodule Explorer.Factory do
alias Explorer.Market.MarketHistory
alias Explorer.Repo
alias Explorer.Utility.MissingBlockRange
alias Explorer.Utility.{MissingBalanceOfToken, MissingBlockRange}
alias Ueberauth.Strategy.Auth0
alias Ueberauth.Auth.Info
@ -1090,6 +1090,13 @@ defmodule Explorer.Factory do
}
end
def missing_balance_of_token_factory do
%MissingBalanceOfToken{
token_contract_address_hash: insert(:token).contract_address_hash,
block_number: block_number()
}
end
def withdrawal_factory do
block = build(:block)
address = build(:address)

@ -19,7 +19,9 @@ defmodule Indexer.Fetcher.TokenBalance do
require Logger
alias Explorer.Chain
alias Explorer.Chain.Address.{CurrentTokenBalance, TokenBalance}
alias Explorer.Chain.Hash
alias Explorer.Utility.MissingBalanceOfToken
alias Indexer.{BufferedTask, TokenBalances, Tracer}
alias Indexer.Fetcher.TokenBalance.Supervisor, as: TokenBalanceSupervisor
@ -97,6 +99,7 @@ defmodule Indexer.Fetcher.TokenBalance do
result =
entries
|> Enum.map(&format_params/1)
|> MissingBalanceOfToken.filter_token_balances_params()
|> increase_retries_count()
|> fetch_from_blockchain()
|> import_token_balances()
@ -130,7 +133,10 @@ defmodule Indexer.Fetcher.TokenBalance do
if Enum.empty?(failed_token_balances) do
{:halt, all_token_balances}
else
failed_token_balances = increase_retries_count(failed_token_balances)
failed_token_balances =
failed_token_balances
|> handle_failed_balances()
|> increase_retries_count()
token_balances_updated_retries_count =
all_token_balances
@ -143,6 +149,23 @@ defmodule Indexer.Fetcher.TokenBalance do
fetched_token_balances
end
defp handle_failed_balances(failed_token_balances) do
{missing_balance_of_balances, other_failed_balances} =
Enum.split_with(failed_token_balances, fn
%{error: error} when is_binary(error) -> error =~ "execution reverted"
_ -> false
end)
MissingBalanceOfToken.insert_from_params(missing_balance_of_balances)
Enum.each(missing_balance_of_balances, fn balance ->
TokenBalance.delete_placeholders_below(balance.token_contract_address_hash, balance.block_number)
CurrentTokenBalance.delete_placeholders_below(balance.token_contract_address_hash, balance.block_number)
end)
other_failed_balances
end
defp increase_retries_count(params_list) do
params_list
|> Enum.map(&Map.put(&1, :retries_count, &1.retries_count + 1))

@ -41,7 +41,7 @@ defmodule Indexer.TokenBalances do
* `token_type` - type of the token that balance belongs to
* `token_id` - token id for ERC-1155/ERC-404 tokens
"""
def fetch_token_balances_from_blockchain([]), do: {:ok, []}
def fetch_token_balances_from_blockchain([]), do: {:ok, %{fetched_token_balances: [], failed_token_balances: []}}
@decorate span(tracer: Tracer)
def fetch_token_balances_from_blockchain(token_balances) do

@ -5,6 +5,8 @@ defmodule Indexer.Fetcher.TokenBalanceTest do
import Mox
alias Explorer.Chain.{Address, Hash}
alias Explorer.Repo
alias Explorer.Utility.MissingBalanceOfToken
alias Indexer.Fetcher.TokenBalance
@moduletag :capture_log
@ -62,7 +64,7 @@ defmodule Indexer.Fetcher.TokenBalanceTest do
nil
) == :ok
token_balance_updated = Explorer.Repo.get_by(Address.TokenBalance, address_hash: address_hash)
token_balance_updated = Repo.get_by(Address.TokenBalance, address_hash: address_hash)
assert token_balance_updated.value == Decimal.new(1_000_000_000_000_000_000_000_000)
assert token_balance_updated.value_fetched_at != nil
@ -110,7 +112,7 @@ defmodule Indexer.Fetcher.TokenBalanceTest do
nil
) == :ok
token_balance_updated = Explorer.Repo.get_by(Address.TokenBalance, address_hash: address_hash)
token_balance_updated = Repo.get_by(Address.TokenBalance, address_hash: address_hash)
assert token_balance_updated.value == Decimal.new(1_000_000_000_000_000_000_000_000)
assert token_balance_updated.value_fetched_at != nil
@ -180,7 +182,62 @@ defmodule Indexer.Fetcher.TokenBalanceTest do
assert 1 =
from(tb in Address.TokenBalance, where: tb.address_hash == ^address_hash)
|> Explorer.Repo.aggregate(:count, :id)
|> Repo.aggregate(:count, :id)
end
test "filters out params with tokens that doesn't implement balanceOf function" do
address = insert(:address)
missing_balance_of_token = insert(:missing_balance_of_token)
assert TokenBalance.run(
[
{address.hash.bytes, missing_balance_of_token.token_contract_address_hash.bytes,
missing_balance_of_token.block_number, "ERC-20", nil, 0}
],
nil
) == :ok
assert Repo.all(Address.TokenBalance) == []
end
test "in case of error deletes token balance placeholders below the given number and inserts new missing balanceOf tokens" do
address = insert(:address)
%{contract_address_hash: token_contract_address_hash} = insert(:token)
insert(:token_balance,
token_contract_address_hash: token_contract_address_hash,
address: address,
block_number: 0,
value_fetched_at: nil,
value: nil
)
expect(
EthereumJSONRPC.Mox,
:json_rpc,
fn [%{id: id, method: "eth_call", params: [%{data: _, to: _}, _]}], _options ->
{:ok,
[
%{
id: id,
jsonrpc: "2.0",
error: %{code: "-32000", message: "execution reverted"}
}
]}
end
)
assert TokenBalance.run(
[
{address.hash.bytes, token_contract_address_hash.bytes, 1, "ERC-20", nil, 0}
],
nil
) == :ok
assert %{token_contract_address_hash: ^token_contract_address_hash, block_number: 1} =
Repo.one(MissingBalanceOfToken)
assert Repo.all(Address.TokenBalance) == []
end
end

@ -337,6 +337,7 @@
"malihu",
"mallowance",
"maxlength",
"mbot",
"mcap",
"mconst",
"mdef",

Loading…
Cancel
Save