Don't fetch internal transactions for simple token transfers

Resolves #1303

Don't fetch internal transactions for transactions that have token
transfers from the logs if the status is `:ok` and the transaction did
not create a contract.

apps/explorer/lib/explorer/chain/import/runner/transactions.ex

@@ -9,6 +9,7 @@ defmodule Explorer.Chain.Import.Runner.Transactions do
 
   alias Ecto.{Multi, Repo}
   alias Explorer.Chain.{Data, Hash, Import, Transaction}
+  alias Explorer.Chain.Import.Runner.TokenTransfers
 
   @behaviour Import.Runner
 
@@ -39,6 +40,7 @@ defmodule Explorer.Chain.Import.Runner.Transactions do
       |> Map.take(~w(on_conflict timeout)a)
       |> Map.put_new(:timeout, @timeout)
       |> Map.put(:timestamps, timestamps)
+      |> Map.put(:token_transfer_transaction_hash_set, token_transfer_transaction_hash_set(options))
 
     Multi.run(multi, :transactions, fn repo, _ ->
       insert(repo, changes_list, insert_options)
@@ -48,18 +50,33 @@ defmodule Explorer.Chain.Import.Runner.Transactions do
   @impl Import.Runner
   def timeout, do: @timeout
 
+  defp token_transfer_transaction_hash_set(options) do
+    token_transfers_params = options[TokenTransfers.option_key()][:params] || []
+
+    MapSet.new(token_transfers_params, & &1.transaction_hash)
+  end
+
   @spec insert(Repo.t(), [map()], %{
           optional(:on_conflict) => Import.Runner.on_conflict(),
           required(:timeout) => timeout,
-          required(:timestamps) => Import.timestamps()
+          required(:timestamps) => Import.timestamps(),
+          required(:token_transfer_transaction_hash_set) => MapSet.t()
         }) :: {:ok, [Hash.t()]}
-  defp insert(repo, changes_list, %{timeout: timeout, timestamps: %{inserted_at: inserted_at} = timestamps} = options)
+  defp insert(
+         repo,
+         changes_list,
+         %{
+           timeout: timeout,
+           timestamps: %{inserted_at: inserted_at} = timestamps,
+           token_transfer_transaction_hash_set: token_transfer_transaction_hash_set
+         } = options
+       )
        when is_list(changes_list) do
     on_conflict = Map.get_lazy(options, :on_conflict, &default_on_conflict/0)
 
     ordered_changes_list =
       changes_list
-      |> timestamp_ok_value_transfers(inserted_at)
+      |> put_internal_transactions_indexed_at(inserted_at, token_transfer_transaction_hash_set)
       # order so that row ShareLocks are grabbed in a consistent order
       |> Enum.sort_by(& &1.hash)
 
@@ -130,15 +147,35 @@ defmodule Explorer.Chain.Import.Runner.Transactions do
     )
   end
 
-  defp timestamp_ok_value_transfers(changes_list, timestamp) when is_list(changes_list) do
+  defp put_internal_transactions_indexed_at(changes_list, timestamp, token_transfer_transaction_hash_set)
-    Enum.map(changes_list, &timestamp_ok_value_transfer(&1, timestamp))
+       when is_list(changes_list) do
+    Enum.map(changes_list, &put_internal_transactions_indexed_at(&1, timestamp, token_transfer_transaction_hash_set))
+  end
+
+  defp put_internal_transactions_indexed_at(%{hash: hash} = changes, timestamp, token_transfer_transaction_hash_set) do
+    token_transfer? = to_string(hash) in token_transfer_transaction_hash_set
+
+    if put_internal_transactions_indexed_at?(changes, token_transfer?) do
+      Map.put(changes, :internal_transactions_indexed_at, timestamp)
+    else
+      changes
+    end
   end
 
   # A post-Byzantium validated transaction will have a status and if it has no input, it is a value transfer only.
   # Internal transactions are only needed when status is `:error` to set `error`.
-  defp timestamp_ok_value_transfer(%{status: :ok, input: %Data{bytes: <<>>}} = changes, timestamp) do
+  defp put_internal_transactions_indexed_at?(%{status: :ok, input: %Data{bytes: <<>>}}, _), do: true
-    Map.put(changes, :internal_transactions_indexed_at, timestamp)
+
+  # A post-Byzantium validated transaction will have a status and if it transfers tokens, the token transfer is in the
+  # log and the internal transactions.
+  # `created_contract_address_hash` must be `nil` because if a contract is created the internal transactions are needed
+  # to get
+  defp put_internal_transactions_indexed_at?(%{status: :ok} = changes, true) do
+    case Map.fetch(changes, :created_contract_address_hash) do
+      {:ok, created_contract_address_hash} when not is_nil(created_contract_address_hash) -> false
+      :error -> true
+    end
   end
 
-  defp timestamp_ok_value_transfer(changes, _), do: changes
+  defp put_internal_transactions_indexed_at?(_, _), do: false
 end

apps/explorer/lib/explorer/chain/transaction.ex

@@ -99,14 +99,15 @@ defmodule Explorer.Chain.Transaction do
    * `internal_transactions_indexed_at` - when `internal_transactions` were fetched by `Indexer` or when they do not
      need to be fetched at `inserted_at`.
 
-     | `status` | `input`    | `internal_transactions_indexed_at`        | `internal_transactions` | Description                                                                             |
+     | `status` | `contract_creation_address_hash` | `input`    | Token Transfer? | `internal_transactions_indexed_at`        | `internal_transactions` | Description                                                                                         |
-     |----------|------------|-------------------------------------------|-------------------------|-----------------------------------------------------------------------------------------|
+     |----------|----------------------------------|------------|-----------------|-------------------------------------------|-------------------------|-----------------------------------------------------------------------------------------------------|
-     | `:ok`    | Empty      | `inserted_at`                             | Unfetched               | Simple `value` transfer succeeded.  Internal transactions would be same value transfer. |
+     | `:ok`    | `nil`                            | Empty      | Don't Care      | `inserted_at`                             | Unfetched               | Simple `value` transfer transaction succeeded.  Internal transactions would be same value transfer. |
-     | `:ok`    | Non-Empty  | When `internal_transactions` are indexed. | Fetched                 | A contract call that succeeded.                                                         |
+     | `:ok`    | `nil`                            | Don't Care | `true`          | `inserted_at`                             | Unfetched               | Token transfer (from `logs`) that didn't happen during a contract creation.                         |
-     | `:error` | Empty      | When `internal_transactions` are indexed. | Fetched                 | Simple `value` transfer failed. Internal transactions fetched for `error`.              |
+     | `:ok`    | Don't Care                       | Non-Empty  | Don't Care      | When `internal_transactions` are indexed. | Fetched                 | A contract call that succeeded.                                                                     |
-     | `:error` | Non-Empty  | When `internal_transactions` are indexed. | Fetched                 | A contract call that failed.                                                            |
+     | `:error` | nil                              | Empty      | Don't Care      | When `internal_transactions` are indexed. | Fetched                 | Simple `value` transfer transaction failed. Internal transactions fetched for `error`.              |
-     | `nil`    | Don't Care | When `internal_transactions` are indexed. | Depends                 | A pending post-Byzantium transaction will only know its status from  receipt.           |
+     | `:error` | Don't Care                       | Non-Empty  | Don't Care      | When `internal_transactions` are indexed. | Fetched                 | A contract call that failed.                                                                        |
-     | `nil`    | Don't Care | When `internal_transactions` are indexed. | Fetched                 | A pre-Byzantium transaction requires internal transactions to determine status          |
+     | `nil`    | Don't Care                       | Don't Care | Don't Care      | When `internal_transactions` are indexed. | Depends                 | A pending post-Byzantium transaction will only know its status from receipt.                        |
+     | `nil`    | Don't Care                       | Don't Care | Don't Care      | When `internal_transactions` are indexed. | Fetched                 | A pre-Byzantium transaction requires internal transactions to determine status.                     |
    * `logs` - events that occurred while mining the `transaction`.
    * `nonce` - the number of transaction made by the sender prior to this one
    * `r` - the R field of the signature. The (r, s) is the normal output of an ECDSA signature, where r is computed as

apps/explorer/test/explorer/chain/import_test.exs

@@ -306,7 +306,8 @@ defmodule Explorer.Chain.ImportTest do
                         <<83, 189, 136, 72, 114, 222, 62, 72, 134, 146, 136, 27, 174, 236, 38, 46, 123, 149, 35, 77, 57,
                           101, 36, 140, 57, 254, 153, 47, 255, 212, 51, 229>>
                     },
-                    internal_transactions_indexed_at: nil
+                    # because there are successful, non-contract-creation token transfer
+                    internal_transactions_indexed_at: %DateTime{}
                   }
                 ],
                 tokens: [

apps/explorer/test/explorer/chain_test.exs

@@ -1195,7 +1195,8 @@ defmodule Explorer.ChainTest do
                         <<83, 189, 136, 72, 114, 222, 62, 72, 134, 146, 136, 27, 174, 236, 38, 46, 123, 149, 35, 77, 57,
                           101, 36, 140, 57, 254, 153, 47, 255, 212, 51, 229>>
                     },
-                    internal_transactions_indexed_at: nil
+                    # because there are successful, non-contract-creation token transfer
+                    internal_transactions_indexed_at: %DateTime{}
                   }
                 ],
                 tokens: [

apps/indexer/lib/indexer/block/realtime/fetcher.ex

@@ -94,7 +94,8 @@ defmodule Indexer.Block.Realtime.Fetcher do
           address_hash_to_fetched_balance_block_number: address_hash_to_block_number,
           address_token_balances: %{params: address_token_balances_params},
           addresses: %{params: addresses_params},
-          transactions: %{params: transactions_params}
+          transactions: %{params: transactions_params},
+          token_transfers: %{params: token_transfers_params}
         } = options
       ) do
     with {:internal_transactions,
@@ -106,6 +107,7 @@ defmodule Indexer.Block.Realtime.Fetcher do
            {:internal_transactions,
             internal_transactions(block_fetcher, %{
               addresses_params: addresses_params,
+              token_transfers_params: token_transfers_params,
               transactions_params: transactions_params
             })},
          {:balances, {:ok, %{addresses_params: balances_addresses_params, balances_params: balances_params}}} <-
@@ -279,10 +281,14 @@ defmodule Indexer.Block.Realtime.Fetcher do
 
   defp internal_transactions(
          %Block.Fetcher{json_rpc_named_arguments: json_rpc_named_arguments},
-         %{addresses_params: addresses_params, transactions_params: transactions_params}
+         %{
+           addresses_params: addresses_params,
+           token_transfers_params: token_transfers_params,
+           transactions_params: transactions_params
+         }
        ) do
     case transactions_params
-         |> transactions_params_to_fetch_internal_transactions_params()
+         |> transactions_params_to_fetch_internal_transactions_params(token_transfers_params)
          |> EthereumJSONRPC.fetch_internal_transactions(json_rpc_named_arguments) do
       {:ok, internal_transactions_params} ->
         merged_addresses_params =
@@ -301,24 +307,36 @@ defmodule Indexer.Block.Realtime.Fetcher do
     end
   end
 
-  defp transactions_params_to_fetch_internal_transactions_params(transactions_params) do
+  defp transactions_params_to_fetch_internal_transactions_params(transactions_params, token_transfers_params) do
-    Enum.flat_map(transactions_params, &transaction_params_to_fetch_internal_transaction_params_list/1)
+    token_transfer_transaction_hash_set = MapSet.new(token_transfers_params, & &1.transaction_hash)
-  end
 
-  # Input-less transactions are value-transfers only, so their internal transactions do not need to be indexed
+    Enum.flat_map(
-  defp transaction_params_to_fetch_internal_transaction_params_list(%{input: "0x"}) do
+      transactions_params,
-    []
+      &transaction_params_to_fetch_internal_transaction_params_list(&1, token_transfer_transaction_hash_set)
+    )
   end
 
-  defp transaction_params_to_fetch_internal_transaction_params_list(%{
+  defp transaction_params_to_fetch_internal_transaction_params_list(
-         block_number: block_number,
+         %{block_number: block_number, transaction_index: transaction_index, hash: hash} = transaction_params,
-         hash: hash,
+         token_transfer_transaction_hash_set
-         transaction_index: transaction_index
+       )
-       })
+       when is_integer(block_number) and is_integer(transaction_index) and is_binary(hash) do
-       when is_integer(block_number) do
+    token_transfer? = hash in token_transfer_transaction_hash_set
-    [%{block_number: block_number, hash_data: to_string(hash), transaction_index: transaction_index}]
+
+    if fetch_internal_transactions?(transaction_params, token_transfer?) do
+      [%{block_number: block_number, transaction_index: transaction_index, hash_data: hash}]
+    else
+      []
+    end
   end
 
+  # Input-less transactions are value-transfers only, so their internal transactions do not need to be indexed
+  defp fetch_internal_transactions?(%{status: :ok, created_contract_address_hash: nil, input: "0x"}, _), do: false
+  # Token transfers not transferred during contract creation don't need internal transactions as the token transfers
+  # derive completely from the logs.
+  defp fetch_internal_transactions?(%{status: :ok, created_contract_address_hash: nil}, true), do: false
+  defp fetch_internal_transactions?(_, _), do: true
+
   defp balances(
          %Block.Fetcher{json_rpc_named_arguments: json_rpc_named_arguments},
          %{addresses_params: addresses_params} = options