Fix TokenTotalSupplyOnDemand fetcher; Add FirstTraceOnDemand fetcher; Add fallback values in blocks to on demand fetchers

apps/block_scout_web/lib/block_scout_web/controllers/api/v2/address_controller.ex

@@ -16,7 +16,7 @@ defmodule BlockScoutWeb.API.V2.AddressController do
   alias BlockScoutWeb.API.V2.{BlockView, TransactionView}
   alias Explorer.ExchangeRates.Token
   alias Explorer.{Chain, Market}
-  alias Indexer.Fetcher.TokenBalanceOnDemand
+  alias Indexer.Fetcher.{CoinBalanceOnDemand, TokenBalanceOnDemand}
 
   @transaction_necessity_by_association [
     necessity_by_association: %{
@@ -45,6 +45,8 @@ defmodule BlockScoutWeb.API.V2.AddressController do
     with {:format, {:ok, address_hash}} <- {:format, Chain.string_to_address_hash(address_hash_string)},
          {:ok, false} <- AccessHelpers.restricted_access?(address_hash_string, params),
          {:not_found, {:ok, address}} <- {:not_found, Chain.hash_to_address(address_hash)} do
+      CoinBalanceOnDemand.trigger_fetch(address)
+
       conn
       |> put_status(200)
       |> render(:address, %{address: address})

apps/block_scout_web/lib/block_scout_web/controllers/api/v2/token_controller.ex

@@ -4,6 +4,7 @@ defmodule BlockScoutWeb.API.V2.TokenController do
   alias BlockScoutWeb.AccessHelpers
   alias BlockScoutWeb.API.V2.TransactionView
   alias Explorer.{Chain, Market}
+  alias Indexer.Fetcher.TokenTotalSupplyOnDemand
 
   import BlockScoutWeb.Chain,
     only: [
@@ -23,6 +24,8 @@ defmodule BlockScoutWeb.API.V2.TokenController do
     with {:format, {:ok, address_hash}} <- {:format, Chain.string_to_address_hash(address_hash_string)},
          {:ok, false} <- AccessHelpers.restricted_access?(address_hash_string, params),
          {:not_found, {:ok, token}} <- {:not_found, Chain.token_from_address_hash(address_hash)} do
+      TokenTotalSupplyOnDemand.trigger_fetch(address_hash)
+
       conn
       |> put_status(200)
       |> render(:token, %{token: Market.add_price(token)})

apps/block_scout_web/lib/block_scout_web/controllers/api/v2/transaction_controller.ex

@@ -15,8 +15,7 @@ defmodule BlockScoutWeb.API.V2.TransactionController do
 
   alias BlockScoutWeb.AccessHelpers
   alias Explorer.Chain
-  alias Explorer.Chain.Import
+  alias Indexer.Fetcher.FirstTraceOnDemand
-  alias Explorer.Chain.Import.Runner.InternalTransactions
 
   action_fallback(BlockScoutWeb.API.V2.FallbackController)
 
@@ -118,42 +117,9 @@ defmodule BlockScoutWeb.API.V2.TransactionController do
             trace.index == 0
           end)
 
-        json_rpc_named_arguments = Application.get_env(:explorer, :json_rpc_named_arguments)
+        if !first_trace_exists do
-
+          FirstTraceOnDemand.trigger_fetch(transaction)
-        internal_transactions =
+        end
-          if first_trace_exists do
-            internal_transactions
-          else
-            response =
-              Chain.fetch_first_trace(
-                [
-                  %{
-                    block_hash: transaction.block_hash,
-                    block_number: transaction.block_number,
-                    hash_data: transaction_hash_string,
-                    transaction_index: transaction.index
-                  }
-                ],
-                json_rpc_named_arguments
-              )
-
-            case response do
-              {:ok, first_trace_params} ->
-                InternalTransactions.run_insert_only(first_trace_params, %{
-                  timeout: :infinity,
-                  timestamps: Import.timestamps(),
-                  internal_transactions: %{params: first_trace_params}
-                })
-
-                Chain.all_transaction_to_internal_transactions(transaction_hash)
-
-              {:error, _} ->
-                internal_transactions
-
-              :ignore ->
-                internal_transactions
-            end
-          end
 
         conn
         |> put_status(200)

apps/block_scout_web/lib/block_scout_web/controllers/transaction_raw_trace_controller.ex

@@ -8,9 +8,8 @@ defmodule BlockScoutWeb.TransactionRawTraceController do
   alias BlockScoutWeb.{AccessHelpers, TransactionController}
   alias EthereumJSONRPC
   alias Explorer.{Chain, Market}
-  alias Explorer.Chain.Import
-  alias Explorer.Chain.Import.Runner.InternalTransactions
   alias Explorer.ExchangeRates.Token
+  alias Indexer.Fetcher.FirstTraceOnDemand
 
   def index(conn, %{"transaction_id" => hash_string} = params) do
     with {:ok, hash} <- Chain.string_to_transaction_hash(hash_string),
@@ -38,42 +37,9 @@ defmodule BlockScoutWeb.TransactionRawTraceController do
             trace.index == 0
           end)
 
-        json_rpc_named_arguments = Application.get_env(:explorer, :json_rpc_named_arguments)
+        if !first_trace_exists do
-
+          FirstTraceOnDemand.trigger_fetch(transaction)
-        internal_transactions =
+        end
-          if first_trace_exists do
-            internal_transactions
-          else
-            response =
-              Chain.fetch_first_trace(
-                [
-                  %{
-                    block_hash: transaction.block_hash,
-                    block_number: transaction.block_number,
-                    hash_data: hash_string,
-                    transaction_index: transaction.index
-                  }
-                ],
-                json_rpc_named_arguments
-              )
-
-            case response do
-              {:ok, first_trace_params} ->
-                InternalTransactions.run_insert_only(first_trace_params, %{
-                  timeout: :infinity,
-                  timestamps: Import.timestamps(),
-                  internal_transactions: %{params: first_trace_params}
-                })
-
-                Chain.all_transaction_to_internal_transactions(hash)
-
-              {:error, _} ->
-                internal_transactions
-
-              :ignore ->
-                internal_transactions
-            end
-          end
 
         render_raw_trace(conn, internal_transactions, transaction, hash)
       end

apps/block_scout_web/lib/block_scout_web/views/block_view.ex

@@ -56,6 +56,7 @@ defmodule BlockScoutWeb.BlockView do
   def show_reward?(_), do: true
 
   def block_reward_text(%Reward{address_hash: beneficiary_address, address_type: :validator}, block_miner_address) do
+    # false here
     if Application.get_env(:explorer, Explorer.Chain.Block.Reward, %{})[:keys_manager_contract_address] do
       %{payout_key: block_miner_payout_address} = Reward.get_validator_payout_key_by_mining(block_miner_address)
 

apps/explorer/lib/explorer/chain.ex

@@ -1952,7 +1952,7 @@ defmodule Explorer.Chain do
   defp check_bytecode_matching(address, _) do
     now = DateTime.utc_now()
     json_rpc_named_arguments = Application.get_env(:explorer, :json_rpc_named_arguments)
-
+    # 'false and' here
     if !address.smart_contract.is_changed_bytecode and
          address.smart_contract.bytecode_checked_at
          |> DateTime.add(@check_bytecode_interval, :second)

apps/explorer/lib/explorer/smart_contract/reader.ex

@@ -107,16 +107,6 @@ defmodule Explorer.SmartContract.Reader do
   * `:json_rpc_named_arguments` - Options to forward for calling the Ethereum JSON RPC. See
     `t:EthereumJSONRPC.json_rpc_named_arguments.t/0` for full list of options.
   """
-  @spec query_contract(
-          String.t(),
-          term(),
-          functions(),
-          true | false
-        ) :: functions_results()
-  def query_contract(contract_address, abi, functions, leave_error_as_map) do
-    query_contract_inner(contract_address, abi, functions, nil, nil, leave_error_as_map)
-  end
-
   @spec query_contract(
           String.t(),
           String.t() | nil,
@@ -124,7 +114,7 @@ defmodule Explorer.SmartContract.Reader do
           functions(),
           true | false
         ) :: functions_results()
-  def query_contract(contract_address, from, abi, functions, leave_error_as_map) do
+  def query_contract(contract_address, from \\ nil, abi, functions, leave_error_as_map) do
     query_contract_inner(contract_address, abi, functions, nil, from, leave_error_as_map)
   end
 

apps/indexer/lib/indexer/application.ex

@@ -5,7 +5,7 @@ defmodule Indexer.Application do
 
   use Application
 
-  alias Indexer.Fetcher.{CoinBalanceOnDemand, TokenTotalSupplyOnDemand}
+  alias Indexer.Fetcher.{CoinBalanceOnDemand, TokenTotalSupplyOnDemand, FirstTraceOnDemand}
   alias Indexer.Memory
   alias Indexer.Prometheus.PendingBlockOperationsCollector
   alias Prometheus.Registry
@@ -27,7 +27,8 @@ defmodule Indexer.Application do
     base_children = [
       {Memory.Monitor, [memory_monitor_options, [name: memory_monitor_name]]},
       {CoinBalanceOnDemand.Supervisor, [json_rpc_named_arguments]},
-      {TokenTotalSupplyOnDemand.Supervisor, [json_rpc_named_arguments]}
+      {TokenTotalSupplyOnDemand.Supervisor, []},
+      {FirstTraceOnDemand.Supervisor, [json_rpc_named_arguments]}
     ]
 
     children =

apps/indexer/lib/indexer/fetcher/coin_balance_on_demand.ex

@@ -233,7 +233,8 @@ defmodule Indexer.Fetcher.CoinBalanceOnDemand do
   defp stale_balance_window(block_number) do
     case AverageBlockTime.average_block_time() do
       {:error, :disabled} ->
-        {:error, :no_average_block_time}
+        fallback_treshold_in_blocks = Application.get_env(:indexer, __MODULE__)[:fallback_treshold_in_blocks]
+        block_number - fallback_treshold_in_blocks
 
       duration ->
         average_block_time =

apps/indexer/lib/indexer/fetcher/first_trace_on_demand.ex

@@ -0,0 +1,85 @@
+defmodule Indexer.Fetcher.FirstTraceOnDemand do
+  @moduledoc """
+    On demand fetcher of first transaction's trace
+  """
+
+  use GenServer
+  use Indexer.Fetcher, restart: :permanent
+
+  alias Explorer.Chain
+  alias Explorer.Chain.Import
+  alias Explorer.Chain.Import.Runner.InternalTransactions
+  alias Explorer.Counters.Helper
+
+  require Logger
+
+  # cache needed to keep track of transactions which are already being processed
+  @cache_name :first_traces_processing
+
+  def trigger_fetch(transaction) do
+    GenServer.cast(__MODULE__, {:fetch, transaction})
+  end
+
+  def fetch_first_trace(transaction, state) do
+    hash_string = to_string(transaction.hash)
+
+    response =
+      Chain.fetch_first_trace(
+        [
+          %{
+            block_hash: transaction.block_hash,
+            block_number: transaction.block_number,
+            hash_data: hash_string,
+            transaction_index: transaction.index
+          }
+        ],
+        state.json_rpc_named_arguments
+      )
+
+    case response do
+      {:ok, first_trace_params} ->
+        InternalTransactions.run_insert_only(first_trace_params, %{
+          timeout: :infinity,
+          timestamps: Import.timestamps(),
+          internal_transactions: %{params: first_trace_params}
+        })
+
+      {:error, reason} ->
+        Logger.error(fn ->
+          ["Error while fetching first trace for tx: #{hash_string} error reason: ", reason]
+        end)
+
+      :ignore ->
+        :ignore
+    end
+
+    :ets.delete(@cache_name, hash_string)
+  end
+
+  def start_link([init_opts, server_opts]) do
+    GenServer.start_link(__MODULE__, init_opts, server_opts)
+  end
+
+  @impl true
+  def init(json_rpc_named_arguments) do
+    Helper.create_cache_table(@cache_name)
+
+    {:ok, %{json_rpc_named_arguments: json_rpc_named_arguments}}
+  end
+
+  @impl true
+  def handle_cast({:fetch, transaction}, state) do
+    hash_string = to_string(transaction.hash)
+
+    case Helper.fetch_from_cache(hash_string, @cache_name) do
+      0 ->
+        :ets.insert(@cache_name, {hash_string, 1})
+        fetch_first_trace(transaction, state)
+
+      1 ->
+        :ignore
+    end
+
+    {:noreply, state}
+  end
+end

apps/indexer/lib/indexer/fetcher/token_balance_on_demand.ex

@@ -93,7 +93,8 @@ defmodule Indexer.Fetcher.TokenBalanceOnDemand do
   defp stale_balance_window(block_number) do
     case AverageBlockTime.average_block_time() do
       {:error, :disabled} ->
-        {:error, :no_average_block_time}
+        fallback_treshold_in_blocks = Application.get_env(:indexer, __MODULE__)[:fallback_treshold_in_blocks]
+        block_number - fallback_treshold_in_blocks
 
       duration ->
         average_block_time =

apps/indexer/lib/indexer/fetcher/token_total_supply_on_demand.ex

@@ -1,7 +1,6 @@
 defmodule Indexer.Fetcher.TokenTotalSupplyOnDemand do
   @moduledoc """
-  Ensures that we have a reasonably up to date token supply.
+    Ensures that we have a reasonably up to date token supply.
-
   """
 
   use GenServer
@@ -15,7 +14,7 @@ defmodule Indexer.Fetcher.TokenTotalSupplyOnDemand do
 
   @spec trigger_fetch(Address.t()) :: :ok
   def trigger_fetch(address) do
-    do_trigger_fetch(address)
+    GenServer.cast(__MODULE__, {:fetch_and_update, address})
   end
 
   ## Callbacks
@@ -24,10 +23,18 @@ defmodule Indexer.Fetcher.TokenTotalSupplyOnDemand do
     GenServer.start_link(__MODULE__, init_opts, server_opts)
   end
 
+  @impl true
   def init(init_arg) do
     {:ok, init_arg}
   end
 
+  @impl true
+  def handle_cast({:fetch_and_update, address}, state) do
+    do_trigger_fetch(address)
+
+    {:noreply, state}
+  end
+
   ## Implementation
 
   defp do_trigger_fetch(address) when not is_nil(address) do

config/runtime.exs

@@ -522,7 +522,9 @@ token_balance_on_demand_fetcher_threshold =
     _ -> 60
   end
 
-config :indexer, Indexer.Fetcher.TokenBalanceOnDemand, threshold: token_balance_on_demand_fetcher_threshold
+config :indexer, Indexer.Fetcher.TokenBalanceOnDemand,
+  threshold: token_balance_on_demand_fetcher_threshold,
+  fallback_treshold_in_blocks: 500
 
 coin_balance_on_demand_fetcher_threshold_minutes = System.get_env("COIN_BALANCE_ON_DEMAND_FETCHER_THRESHOLD_MINUTES")
 
@@ -533,7 +535,9 @@ coin_balance_on_demand_fetcher_threshold =
     _ -> 60
   end
 
-config :indexer, Indexer.Fetcher.CoinBalanceOnDemand, threshold: coin_balance_on_demand_fetcher_threshold
+config :indexer, Indexer.Fetcher.CoinBalanceOnDemand,
+  threshold: coin_balance_on_demand_fetcher_threshold,
+  fallback_treshold_in_blocks: 500
 
 config :indexer, Indexer.Fetcher.BlockReward.Supervisor,
   disabled?: System.get_env("INDEXER_DISABLE_BLOCK_REWARD_FETCHER", "false") == "true"