Fix TokenTotalSupplyOnDemand fetcher; Add FirstTraceOnDemand fetcher; Add fallback values in blocks to on demand fetchers

pull/6933/head
Nikita Pozdniakov 2 years ago
parent d729f80038
commit ca59693fd3
No known key found for this signature in database
GPG Key ID: F344106F9804FE5F
  1. 4
      apps/block_scout_web/lib/block_scout_web/controllers/api/v2/address_controller.ex
  2. 3
      apps/block_scout_web/lib/block_scout_web/controllers/api/v2/token_controller.ex
  3. 42
      apps/block_scout_web/lib/block_scout_web/controllers/api/v2/transaction_controller.ex
  4. 42
      apps/block_scout_web/lib/block_scout_web/controllers/transaction_raw_trace_controller.ex
  5. 1
      apps/block_scout_web/lib/block_scout_web/views/block_view.ex
  6. 2
      apps/explorer/lib/explorer/chain.ex
  7. 12
      apps/explorer/lib/explorer/smart_contract/reader.ex
  8. 5
      apps/indexer/lib/indexer/application.ex
  9. 3
      apps/indexer/lib/indexer/fetcher/coin_balance_on_demand.ex
  10. 85
      apps/indexer/lib/indexer/fetcher/first_trace_on_demand.ex
  11. 3
      apps/indexer/lib/indexer/fetcher/token_balance_on_demand.ex
  12. 13
      apps/indexer/lib/indexer/fetcher/token_total_supply_on_demand.ex
  13. 8
      config/runtime.exs

@ -16,7 +16,7 @@ defmodule BlockScoutWeb.API.V2.AddressController do
alias BlockScoutWeb.API.V2.{BlockView, TransactionView}
alias Explorer.ExchangeRates.Token
alias Explorer.{Chain, Market}
alias Indexer.Fetcher.TokenBalanceOnDemand
alias Indexer.Fetcher.{CoinBalanceOnDemand, TokenBalanceOnDemand}
@transaction_necessity_by_association [
necessity_by_association: %{
@ -45,6 +45,8 @@ defmodule BlockScoutWeb.API.V2.AddressController do
with {:format, {:ok, address_hash}} <- {:format, Chain.string_to_address_hash(address_hash_string)},
{:ok, false} <- AccessHelpers.restricted_access?(address_hash_string, params),
{:not_found, {:ok, address}} <- {:not_found, Chain.hash_to_address(address_hash)} do
CoinBalanceOnDemand.trigger_fetch(address)
conn
|> put_status(200)
|> render(:address, %{address: address})

@ -4,6 +4,7 @@ defmodule BlockScoutWeb.API.V2.TokenController do
alias BlockScoutWeb.AccessHelpers
alias BlockScoutWeb.API.V2.TransactionView
alias Explorer.{Chain, Market}
alias Indexer.Fetcher.TokenTotalSupplyOnDemand
import BlockScoutWeb.Chain,
only: [
@ -23,6 +24,8 @@ defmodule BlockScoutWeb.API.V2.TokenController do
with {:format, {:ok, address_hash}} <- {:format, Chain.string_to_address_hash(address_hash_string)},
{:ok, false} <- AccessHelpers.restricted_access?(address_hash_string, params),
{:not_found, {:ok, token}} <- {:not_found, Chain.token_from_address_hash(address_hash)} do
TokenTotalSupplyOnDemand.trigger_fetch(address_hash)
conn
|> put_status(200)
|> render(:token, %{token: Market.add_price(token)})

@ -15,8 +15,7 @@ defmodule BlockScoutWeb.API.V2.TransactionController do
alias BlockScoutWeb.AccessHelpers
alias Explorer.Chain
alias Explorer.Chain.Import
alias Explorer.Chain.Import.Runner.InternalTransactions
alias Indexer.Fetcher.FirstTraceOnDemand
action_fallback(BlockScoutWeb.API.V2.FallbackController)
@ -118,42 +117,9 @@ defmodule BlockScoutWeb.API.V2.TransactionController do
trace.index == 0
end)
json_rpc_named_arguments = Application.get_env(:explorer, :json_rpc_named_arguments)
internal_transactions =
if first_trace_exists do
internal_transactions
else
response =
Chain.fetch_first_trace(
[
%{
block_hash: transaction.block_hash,
block_number: transaction.block_number,
hash_data: transaction_hash_string,
transaction_index: transaction.index
}
],
json_rpc_named_arguments
)
case response do
{:ok, first_trace_params} ->
InternalTransactions.run_insert_only(first_trace_params, %{
timeout: :infinity,
timestamps: Import.timestamps(),
internal_transactions: %{params: first_trace_params}
})
Chain.all_transaction_to_internal_transactions(transaction_hash)
{:error, _} ->
internal_transactions
:ignore ->
internal_transactions
end
end
if !first_trace_exists do
FirstTraceOnDemand.trigger_fetch(transaction)
end
conn
|> put_status(200)

@ -8,9 +8,8 @@ defmodule BlockScoutWeb.TransactionRawTraceController do
alias BlockScoutWeb.{AccessHelpers, TransactionController}
alias EthereumJSONRPC
alias Explorer.{Chain, Market}
alias Explorer.Chain.Import
alias Explorer.Chain.Import.Runner.InternalTransactions
alias Explorer.ExchangeRates.Token
alias Indexer.Fetcher.FirstTraceOnDemand
def index(conn, %{"transaction_id" => hash_string} = params) do
with {:ok, hash} <- Chain.string_to_transaction_hash(hash_string),
@ -38,42 +37,9 @@ defmodule BlockScoutWeb.TransactionRawTraceController do
trace.index == 0
end)
json_rpc_named_arguments = Application.get_env(:explorer, :json_rpc_named_arguments)
internal_transactions =
if first_trace_exists do
internal_transactions
else
response =
Chain.fetch_first_trace(
[
%{
block_hash: transaction.block_hash,
block_number: transaction.block_number,
hash_data: hash_string,
transaction_index: transaction.index
}
],
json_rpc_named_arguments
)
case response do
{:ok, first_trace_params} ->
InternalTransactions.run_insert_only(first_trace_params, %{
timeout: :infinity,
timestamps: Import.timestamps(),
internal_transactions: %{params: first_trace_params}
})
Chain.all_transaction_to_internal_transactions(hash)
{:error, _} ->
internal_transactions
:ignore ->
internal_transactions
end
end
if !first_trace_exists do
FirstTraceOnDemand.trigger_fetch(transaction)
end
render_raw_trace(conn, internal_transactions, transaction, hash)
end

@ -56,6 +56,7 @@ defmodule BlockScoutWeb.BlockView do
def show_reward?(_), do: true
def block_reward_text(%Reward{address_hash: beneficiary_address, address_type: :validator}, block_miner_address) do
# false here
if Application.get_env(:explorer, Explorer.Chain.Block.Reward, %{})[:keys_manager_contract_address] do
%{payout_key: block_miner_payout_address} = Reward.get_validator_payout_key_by_mining(block_miner_address)

@ -1952,7 +1952,7 @@ defmodule Explorer.Chain do
defp check_bytecode_matching(address, _) do
now = DateTime.utc_now()
json_rpc_named_arguments = Application.get_env(:explorer, :json_rpc_named_arguments)
# 'false and' here
if !address.smart_contract.is_changed_bytecode and
address.smart_contract.bytecode_checked_at
|> DateTime.add(@check_bytecode_interval, :second)

@ -107,16 +107,6 @@ defmodule Explorer.SmartContract.Reader do
* `:json_rpc_named_arguments` - Options to forward for calling the Ethereum JSON RPC. See
`t:EthereumJSONRPC.json_rpc_named_arguments.t/0` for full list of options.
"""
@spec query_contract(
String.t(),
term(),
functions(),
true | false
) :: functions_results()
def query_contract(contract_address, abi, functions, leave_error_as_map) do
query_contract_inner(contract_address, abi, functions, nil, nil, leave_error_as_map)
end
@spec query_contract(
String.t(),
String.t() | nil,
@ -124,7 +114,7 @@ defmodule Explorer.SmartContract.Reader do
functions(),
true | false
) :: functions_results()
def query_contract(contract_address, from, abi, functions, leave_error_as_map) do
def query_contract(contract_address, from \\ nil, abi, functions, leave_error_as_map) do
query_contract_inner(contract_address, abi, functions, nil, from, leave_error_as_map)
end

@ -5,7 +5,7 @@ defmodule Indexer.Application do
use Application
alias Indexer.Fetcher.{CoinBalanceOnDemand, TokenTotalSupplyOnDemand}
alias Indexer.Fetcher.{CoinBalanceOnDemand, TokenTotalSupplyOnDemand, FirstTraceOnDemand}
alias Indexer.Memory
alias Indexer.Prometheus.PendingBlockOperationsCollector
alias Prometheus.Registry
@ -27,7 +27,8 @@ defmodule Indexer.Application do
base_children = [
{Memory.Monitor, [memory_monitor_options, [name: memory_monitor_name]]},
{CoinBalanceOnDemand.Supervisor, [json_rpc_named_arguments]},
{TokenTotalSupplyOnDemand.Supervisor, [json_rpc_named_arguments]}
{TokenTotalSupplyOnDemand.Supervisor, []},
{FirstTraceOnDemand.Supervisor, [json_rpc_named_arguments]}
]
children =

@ -233,7 +233,8 @@ defmodule Indexer.Fetcher.CoinBalanceOnDemand do
defp stale_balance_window(block_number) do
case AverageBlockTime.average_block_time() do
{:error, :disabled} ->
{:error, :no_average_block_time}
fallback_treshold_in_blocks = Application.get_env(:indexer, __MODULE__)[:fallback_treshold_in_blocks]
block_number - fallback_treshold_in_blocks
duration ->
average_block_time =

@ -0,0 +1,85 @@
defmodule Indexer.Fetcher.FirstTraceOnDemand do
@moduledoc """
On demand fetcher of first transaction's trace
"""
use GenServer
use Indexer.Fetcher, restart: :permanent
alias Explorer.Chain
alias Explorer.Chain.Import
alias Explorer.Chain.Import.Runner.InternalTransactions
alias Explorer.Counters.Helper
require Logger
# cache needed to keep track of transactions which are already being processed
@cache_name :first_traces_processing
def trigger_fetch(transaction) do
GenServer.cast(__MODULE__, {:fetch, transaction})
end
def fetch_first_trace(transaction, state) do
hash_string = to_string(transaction.hash)
response =
Chain.fetch_first_trace(
[
%{
block_hash: transaction.block_hash,
block_number: transaction.block_number,
hash_data: hash_string,
transaction_index: transaction.index
}
],
state.json_rpc_named_arguments
)
case response do
{:ok, first_trace_params} ->
InternalTransactions.run_insert_only(first_trace_params, %{
timeout: :infinity,
timestamps: Import.timestamps(),
internal_transactions: %{params: first_trace_params}
})
{:error, reason} ->
Logger.error(fn ->
["Error while fetching first trace for tx: #{hash_string} error reason: ", reason]
end)
:ignore ->
:ignore
end
:ets.delete(@cache_name, hash_string)
end
def start_link([init_opts, server_opts]) do
GenServer.start_link(__MODULE__, init_opts, server_opts)
end
@impl true
def init(json_rpc_named_arguments) do
Helper.create_cache_table(@cache_name)
{:ok, %{json_rpc_named_arguments: json_rpc_named_arguments}}
end
@impl true
def handle_cast({:fetch, transaction}, state) do
hash_string = to_string(transaction.hash)
case Helper.fetch_from_cache(hash_string, @cache_name) do
0 ->
:ets.insert(@cache_name, {hash_string, 1})
fetch_first_trace(transaction, state)
1 ->
:ignore
end
{:noreply, state}
end
end

@ -93,7 +93,8 @@ defmodule Indexer.Fetcher.TokenBalanceOnDemand do
defp stale_balance_window(block_number) do
case AverageBlockTime.average_block_time() do
{:error, :disabled} ->
{:error, :no_average_block_time}
fallback_treshold_in_blocks = Application.get_env(:indexer, __MODULE__)[:fallback_treshold_in_blocks]
block_number - fallback_treshold_in_blocks
duration ->
average_block_time =

@ -1,7 +1,6 @@
defmodule Indexer.Fetcher.TokenTotalSupplyOnDemand do
@moduledoc """
Ensures that we have a reasonably up to date token supply.
Ensures that we have a reasonably up to date token supply.
"""
use GenServer
@ -15,7 +14,7 @@ defmodule Indexer.Fetcher.TokenTotalSupplyOnDemand do
@spec trigger_fetch(Address.t()) :: :ok
def trigger_fetch(address) do
do_trigger_fetch(address)
GenServer.cast(__MODULE__, {:fetch_and_update, address})
end
## Callbacks
@ -24,10 +23,18 @@ defmodule Indexer.Fetcher.TokenTotalSupplyOnDemand do
GenServer.start_link(__MODULE__, init_opts, server_opts)
end
@impl true
def init(init_arg) do
{:ok, init_arg}
end
@impl true
def handle_cast({:fetch_and_update, address}, state) do
do_trigger_fetch(address)
{:noreply, state}
end
## Implementation
defp do_trigger_fetch(address) when not is_nil(address) do

@ -522,7 +522,9 @@ token_balance_on_demand_fetcher_threshold =
_ -> 60
end
config :indexer, Indexer.Fetcher.TokenBalanceOnDemand, threshold: token_balance_on_demand_fetcher_threshold
config :indexer, Indexer.Fetcher.TokenBalanceOnDemand,
threshold: token_balance_on_demand_fetcher_threshold,
fallback_treshold_in_blocks: 500
coin_balance_on_demand_fetcher_threshold_minutes = System.get_env("COIN_BALANCE_ON_DEMAND_FETCHER_THRESHOLD_MINUTES")
@ -533,7 +535,9 @@ coin_balance_on_demand_fetcher_threshold =
_ -> 60
end
config :indexer, Indexer.Fetcher.CoinBalanceOnDemand, threshold: coin_balance_on_demand_fetcher_threshold
config :indexer, Indexer.Fetcher.CoinBalanceOnDemand,
threshold: coin_balance_on_demand_fetcher_threshold,
fallback_treshold_in_blocks: 500
config :indexer, Indexer.Fetcher.BlockReward.Supervisor,
disabled?: System.get_env("INDEXER_DISABLE_BLOCK_REWARD_FETCHER", "false") == "true"

Loading…
Cancel
Save