Add tracing by block logic for geth

CHANGELOG.md

@@ -4,6 +4,8 @@
 
 ### Features
 
+- [#9072](https://github.com/blockscout/blockscout/pull/9072) - Add tracing by block logic for geth
+
 ### Fixes
 
 - [#9109](https://github.com/blockscout/blockscout/pull/9109) - Return current exchange rate in api/v2/stats

apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/geth.ex

@@ -63,12 +63,65 @@ defmodule EthereumJSONRPC.Geth do
   def fetch_first_trace(_transactions_params, _json_rpc_named_arguments), do: :ignore
 
   @doc """
-  Internal transaction fetching for entire blocks is not currently supported for Geth.
-
-  To signal to the caller that fetching is not supported, `:ignore` is returned.
+  Fetches the `t:Explorer.Chain.InternalTransaction.changeset/2` params from the Geth trace URL.
   """
   @impl EthereumJSONRPC.Variant
-  def fetch_block_internal_transactions(_block_range, _json_rpc_named_arguments), do: :ignore
+  def fetch_block_internal_transactions(block_numbers, json_rpc_named_arguments) do
+    id_to_params = id_to_params(block_numbers)
+
+    with {:ok, blocks_responses} <-
+           id_to_params
+           |> debug_trace_block_by_number_requests()
+           |> json_rpc(json_rpc_named_arguments),
+         :ok <- check_errors_exist(blocks_responses, id_to_params) do
+      transactions_params = to_transactions_params(blocks_responses, id_to_params)
+
+      {transactions_id_to_params, transactions_responses} =
+        Enum.reduce(transactions_params, {%{}, []}, fn {params, calls}, {id_to_params_acc, calls_acc} ->
+          {Map.put(id_to_params_acc, params[:id], params), [calls | calls_acc]}
+        end)
+
+      debug_trace_transaction_responses_to_internal_transactions_params(
+        transactions_responses,
+        transactions_id_to_params,
+        json_rpc_named_arguments
+      )
+    end
+  end
+
+  defp check_errors_exist(blocks_responses, id_to_params) do
+    blocks_responses
+    |> EthereumJSONRPC.sanitize_responses(id_to_params)
+    |> Enum.reduce([], fn
+      %{result: _result}, acc -> acc
+      %{error: error}, acc -> [error | acc]
+    end)
+    |> case do
+      [] -> :ok
+      errors -> {:error, errors}
+    end
+  end
+
+  defp to_transactions_params(blocks_responses, id_to_params) do
+    Enum.reduce(blocks_responses, [], fn %{id: id, result: tx_result}, blocks_acc ->
+      extract_transactions_params(Map.fetch!(id_to_params, id), tx_result) ++ blocks_acc
+    end)
+  end
+
+  defp extract_transactions_params(block_number, tx_result) do
+    tx_result
+    |> Enum.reduce({[], 0}, fn %{"txHash" => tx_hash, "result" => calls_result}, {tx_acc, counter} ->
+      {
+        [
+          {%{block_number: block_number, hash_data: tx_hash, transaction_index: counter, id: counter},
+           %{id: counter, result: calls_result}}
+          | tx_acc
+        ],
+        counter + 1
+      }
+    end)
+    |> elem(0)
+  end
 
   @doc """
   Fetches the pending transactions from the Geth node.
@@ -84,6 +137,10 @@ defmodule EthereumJSONRPC.Geth do
     end)
   end
 
+  defp debug_trace_block_by_number_requests(id_to_params) do
+    Enum.map(id_to_params, &debug_trace_block_by_number_request/1)
+  end
+
   @tracer_path "priv/js/ethereum_jsonrpc/geth/debug_traceTransaction/tracer.js"
   @external_resource @tracer_path
   @tracer File.read!(@tracer_path)
@@ -92,30 +149,39 @@ defmodule EthereumJSONRPC.Geth do
     debug_trace_transaction_timeout =
       Application.get_env(:ethereum_jsonrpc, __MODULE__)[:debug_trace_transaction_timeout]
 
-    tracer =
-      cond do
-        tracer_type() == "js" ->
-          %{"tracer" => @tracer}
-
-        tracer_type() in ~w(opcode polygon_edge) ->
-          %{
-            "enableMemory" => true,
-            "disableStack" => false,
-            "disableStorage" => true,
-            "enableReturnData" => false
-          }
-
-        true ->
-          %{"tracer" => "callTracer"}
-      end
-
     request(%{
       id: id,
       method: "debug_traceTransaction",
-      params: [hash_data, %{timeout: debug_trace_transaction_timeout} |> Map.merge(tracer)]
+      params: [hash_data, %{timeout: debug_trace_transaction_timeout} |> Map.merge(tracer_params())]
+    })
+  end
+
+  defp debug_trace_block_by_number_request({id, block_number}) do
+    request(%{
+      id: id,
+      method: "debug_traceBlockByNumber",
+      params: [integer_to_quantity(block_number), tracer_params()]
     })
   end
 
+  defp tracer_params do
+    cond do
+      tracer_type() == "js" ->
+        %{"tracer" => @tracer}
+
+      tracer_type() in ~w(opcode polygon_edge) ->
+        %{
+          "enableMemory" => true,
+          "disableStack" => false,
+          "disableStorage" => true,
+          "enableReturnData" => false
+        }
+
+      true ->
+        %{"tracer" => "callTracer"}
+    end
+  end
+
   defp debug_trace_transaction_responses_to_internal_transactions_params(
          [%{result: %{"structLogs" => _}} | _] = responses,
          id_to_params,

apps/ethereum_jsonrpc/test/ethereum_jsonrpc/geth_test.exs

@@ -555,8 +555,172 @@ defmodule EthereumJSONRPC.GethTest do
   end
 
   describe "fetch_block_internal_transactions/1" do
-    test "is not supported", %{json_rpc_named_arguments: json_rpc_named_arguments} do
-      EthereumJSONRPC.Geth.fetch_block_internal_transactions([], json_rpc_named_arguments)
+    setup do
+      EthereumJSONRPC.Case.Geth.Mox.setup()
+    end
+
+    test "is supported", %{json_rpc_named_arguments: json_rpc_named_arguments} do
+      block_number = 3_287_375
+      block_quantity = EthereumJSONRPC.integer_to_quantity(block_number)
+      transaction_hash = "0x32b17f27ddb546eab3c4c33f31eb22c1cb992d4ccc50dae26922805b717efe5c"
+
+      expect(EthereumJSONRPC.Mox, :json_rpc, fn [%{id: id, params: [^block_quantity, %{"tracer" => "callTracer"}]}],
+                                                _ ->
+        {:ok,
+         [
+           %{
+             id: id,
+             result: [
+               %{
+                 "result" => %{
+                   "calls" => [
+                     %{
+                       "from" => "0x4200000000000000000000000000000000000015",
+                       "gas" => "0xe9a3c",
+                       "gasUsed" => "0x4a28",
+                       "input" =>
+                         "0x015d8eb900000000000000000000000000000000000000000000000000000000009cb0d80000000000000000000000000000000000000000000000000000000065898738000000000000000000000000000000000000000000000000000000000000001b65f7961a6893850c1f001edeaa0aa4f1fb36b67eee61a8623f8f4da81be25c0000000000000000000000000000000000000000000000000000000000000000050000000000000000000000007431310e026b69bfc676c0013e12a1a11411eec9000000000000000000000000000000000000000000000000000000000000083400000000000000000000000000000000000000000000000000000000000f4240",
+                       "to" => "0x6df83a19647a398d48e77a6835f4a28eb7e2f7c0",
+                       "type" => "DELEGATECALL",
+                       "value" => "0x0"
+                     }
+                   ],
+                   "from" => "0xdeaddeaddeaddeaddeaddeaddeaddeaddead0001",
+                   "gas" => "0xf4240",
+                   "gasUsed" => "0xb6f9",
+                   "input" =>
+                     "0x015d8eb900000000000000000000000000000000000000000000000000000000009cb0d80000000000000000000000000000000000000000000000000000000065898738000000000000000000000000000000000000000000000000000000000000001b65f7961a6893850c1f001edeaa0aa4f1fb36b67eee61a8623f8f4da81be25c0000000000000000000000000000000000000000000000000000000000000000050000000000000000000000007431310e026b69bfc676c0013e12a1a11411eec9000000000000000000000000000000000000000000000000000000000000083400000000000000000000000000000000000000000000000000000000000f4240",
+                   "to" => "0x4200000000000000000000000000000000000015",
+                   "type" => "CALL",
+                   "value" => "0x0"
+                 },
+                 "txHash" => transaction_hash
+               }
+             ]
+           }
+         ]}
+      end)
+
+      Application.put_env(:ethereum_jsonrpc, Geth, tracer: "call_tracer", debug_trace_transaction_timeout: "5s")
+
+      assert {:ok,
+              [
+                %{
+                  block_number: 3_287_375,
+                  call_type: "call",
+                  from_address_hash: "0xdeaddeaddeaddeaddeaddeaddeaddeaddead0001",
+                  gas: 1_000_000,
+                  gas_used: 46841,
+                  index: 0,
+                  input:
+                    "0x015d8eb900000000000000000000000000000000000000000000000000000000009cb0d80000000000000000000000000000000000000000000000000000000065898738000000000000000000000000000000000000000000000000000000000000001b65f7961a6893850c1f001edeaa0aa4f1fb36b67eee61a8623f8f4da81be25c0000000000000000000000000000000000000000000000000000000000000000050000000000000000000000007431310e026b69bfc676c0013e12a1a11411eec9000000000000000000000000000000000000000000000000000000000000083400000000000000000000000000000000000000000000000000000000000f4240",
+                  output: "0x",
+                  to_address_hash: "0x4200000000000000000000000000000000000015",
+                  trace_address: [],
+                  transaction_hash: ^transaction_hash,
+                  transaction_index: 0,
+                  type: "call",
+                  value: 0
+                },
+                %{
+                  block_number: 3_287_375,
+                  call_type: "delegatecall",
+                  from_address_hash: "0x4200000000000000000000000000000000000015",
+                  gas: 956_988,
+                  gas_used: 18984,
+                  index: 1,
+                  input:
+                    "0x015d8eb900000000000000000000000000000000000000000000000000000000009cb0d80000000000000000000000000000000000000000000000000000000065898738000000000000000000000000000000000000000000000000000000000000001b65f7961a6893850c1f001edeaa0aa4f1fb36b67eee61a8623f8f4da81be25c0000000000000000000000000000000000000000000000000000000000000000050000000000000000000000007431310e026b69bfc676c0013e12a1a11411eec9000000000000000000000000000000000000000000000000000000000000083400000000000000000000000000000000000000000000000000000000000f4240",
+                  output: "0x",
+                  to_address_hash: "0x6df83a19647a398d48e77a6835f4a28eb7e2f7c0",
+                  trace_address: [0],
+                  transaction_hash: ^transaction_hash,
+                  transaction_index: 0,
+                  type: "call",
+                  value: 0
+                }
+              ]} = Geth.fetch_block_internal_transactions([block_number], json_rpc_named_arguments)
+    end
+
+    test "result is the same as fetch_internal_transactions/2", %{json_rpc_named_arguments: json_rpc_named_arguments} do
+      block_number = 3_287_375
+      block_quantity = EthereumJSONRPC.integer_to_quantity(block_number)
+      transaction_hash = "0x32b17f27ddb546eab3c4c33f31eb22c1cb992d4ccc50dae26922805b717efe5c"
+
+      expect(EthereumJSONRPC.Mox, :json_rpc, 2, fn
+        [%{id: id, params: [^block_quantity, %{"tracer" => "callTracer"}]}], _ ->
+          {:ok,
+           [
+             %{
+               id: id,
+               result: [
+                 %{
+                   "result" => %{
+                     "calls" => [
+                       %{
+                         "from" => "0x4200000000000000000000000000000000000015",
+                         "gas" => "0xe9a3c",
+                         "gasUsed" => "0x4a28",
+                         "input" =>
+                           "0x015d8eb900000000000000000000000000000000000000000000000000000000009cb0d80000000000000000000000000000000000000000000000000000000065898738000000000000000000000000000000000000000000000000000000000000001b65f7961a6893850c1f001edeaa0aa4f1fb36b67eee61a8623f8f4da81be25c0000000000000000000000000000000000000000000000000000000000000000050000000000000000000000007431310e026b69bfc676c0013e12a1a11411eec9000000000000000000000000000000000000000000000000000000000000083400000000000000000000000000000000000000000000000000000000000f4240",
+                         "to" => "0x6df83a19647a398d48e77a6835f4a28eb7e2f7c0",
+                         "type" => "DELEGATECALL",
+                         "value" => "0x0"
+                       }
+                     ],
+                     "from" => "0xdeaddeaddeaddeaddeaddeaddeaddeaddead0001",
+                     "gas" => "0xf4240",
+                     "gasUsed" => "0xb6f9",
+                     "input" =>
+                       "0x015d8eb900000000000000000000000000000000000000000000000000000000009cb0d80000000000000000000000000000000000000000000000000000000065898738000000000000000000000000000000000000000000000000000000000000001b65f7961a6893850c1f001edeaa0aa4f1fb36b67eee61a8623f8f4da81be25c0000000000000000000000000000000000000000000000000000000000000000050000000000000000000000007431310e026b69bfc676c0013e12a1a11411eec9000000000000000000000000000000000000000000000000000000000000083400000000000000000000000000000000000000000000000000000000000f4240",
+                     "to" => "0x4200000000000000000000000000000000000015",
+                     "type" => "CALL",
+                     "value" => "0x0"
+                   },
+                   "txHash" => transaction_hash
+                 }
+               ]
+             }
+           ]}
+
+        [%{id: id, params: [^transaction_hash, %{"tracer" => "callTracer"}]}], _ ->
+          {:ok,
+           [
+             %{
+               id: id,
+               result: %{
+                 "calls" => [
+                   %{
+                     "from" => "0x4200000000000000000000000000000000000015",
+                     "gas" => "0xe9a3c",
+                     "gasUsed" => "0x4a28",
+                     "input" =>
+                       "0x015d8eb900000000000000000000000000000000000000000000000000000000009cb0d80000000000000000000000000000000000000000000000000000000065898738000000000000000000000000000000000000000000000000000000000000001b65f7961a6893850c1f001edeaa0aa4f1fb36b67eee61a8623f8f4da81be25c0000000000000000000000000000000000000000000000000000000000000000050000000000000000000000007431310e026b69bfc676c0013e12a1a11411eec9000000000000000000000000000000000000000000000000000000000000083400000000000000000000000000000000000000000000000000000000000f4240",
+                     "to" => "0x6df83a19647a398d48e77a6835f4a28eb7e2f7c0",
+                     "type" => "DELEGATECALL",
+                     "value" => "0x0"
+                   }
+                 ],
+                 "from" => "0xdeaddeaddeaddeaddeaddeaddeaddeaddead0001",
+                 "gas" => "0xf4240",
+                 "gasUsed" => "0xb6f9",
+                 "input" =>
+                   "0x015d8eb900000000000000000000000000000000000000000000000000000000009cb0d80000000000000000000000000000000000000000000000000000000065898738000000000000000000000000000000000000000000000000000000000000001b65f7961a6893850c1f001edeaa0aa4f1fb36b67eee61a8623f8f4da81be25c0000000000000000000000000000000000000000000000000000000000000000050000000000000000000000007431310e026b69bfc676c0013e12a1a11411eec9000000000000000000000000000000000000000000000000000000000000083400000000000000000000000000000000000000000000000000000000000f4240",
+                 "to" => "0x4200000000000000000000000000000000000015",
+                 "type" => "CALL",
+                 "value" => "0x0"
+               }
+             }
+           ]}
+      end)
+
+      Application.put_env(:ethereum_jsonrpc, Geth, tracer: "call_tracer", debug_trace_transaction_timeout: "5s")
+
+      assert Geth.fetch_block_internal_transactions([block_number], json_rpc_named_arguments) ==
+               Geth.fetch_internal_transactions(
+                 [%{block_number: block_number, transaction_index: 0, hash_data: transaction_hash}],
+                 json_rpc_named_arguments
+               )
     end
   end
 

apps/indexer/lib/indexer/fetcher/internal_transaction.ex

@@ -111,19 +111,7 @@ defmodule Indexer.Fetcher.InternalTransaction do
 
     json_rpc_named_arguments
     |> Keyword.fetch!(:variant)
-    |> case do
-      variant
-      when variant in [EthereumJSONRPC.Nethermind, EthereumJSONRPC.Erigon, EthereumJSONRPC.Besu, EthereumJSONRPC.RSK] ->
-        EthereumJSONRPC.fetch_block_internal_transactions(filtered_unique_numbers, json_rpc_named_arguments)
-
-      _ ->
-        try do
-          fetch_block_internal_transactions_by_transactions(filtered_unique_numbers, json_rpc_named_arguments)
-        rescue
-          error ->
-            {:error, error, __STACKTRACE__}
-        end
-    end
+    |> fetch_internal_transactions(filtered_unique_numbers, json_rpc_named_arguments)
     |> case do
       {:ok, internal_transactions_params} ->
         safe_import_internal_transaction(internal_transactions_params, filtered_unique_numbers)
@@ -159,6 +147,33 @@ defmodule Indexer.Fetcher.InternalTransaction do
     end
   end
 
+  defp fetch_internal_transactions(variant, block_numbers, json_rpc_named_arguments) do
+    if variant in block_traceable_variants() do
+      EthereumJSONRPC.fetch_block_internal_transactions(block_numbers, json_rpc_named_arguments)
+    else
+      try do
+        fetch_block_internal_transactions_by_transactions(block_numbers, json_rpc_named_arguments)
+      rescue
+        error ->
+          {:error, error, __STACKTRACE__}
+      end
+    end
+  end
+
+  @default_block_traceable_variants [
+    EthereumJSONRPC.Nethermind,
+    EthereumJSONRPC.Erigon,
+    EthereumJSONRPC.Besu,
+    EthereumJSONRPC.RSK
+  ]
+  defp block_traceable_variants do
+    if Application.get_env(:ethereum_jsonrpc, EthereumJSONRPC.Geth)[:block_traceable?] do
+      [EthereumJSONRPC.Geth | @default_block_traceable_variants]
+    else
+      @default_block_traceable_variants
+    end
+  end
+
   defp drop_genesis(block_numbers, json_rpc_named_arguments) do
     first_block = Application.get_env(:indexer, :trace_first_block)
 

config/runtime.exs

@@ -162,6 +162,7 @@ config :ethereum_jsonrpc, EthereumJSONRPC.HTTP,
     |> Map.to_list()
 
 config :ethereum_jsonrpc, EthereumJSONRPC.Geth,
+  block_traceable?: ConfigHelper.parse_bool_env_var("ETHEREUM_JSONRPC_GETH_TRACE_BY_BLOCK"),
   debug_trace_transaction_timeout: System.get_env("ETHEREUM_JSONRPC_DEBUG_TRACE_TRANSACTION_TIMEOUT", "5s"),
   tracer:
     if(ConfigHelper.chain_type() == "polygon_edge",

docker-compose/envs/common-blockscout.env

@@ -18,6 +18,7 @@ ETHEREUM_JSONRPC_DISABLE_ARCHIVE_BALANCES=false
 #ETHEREUM_JSONRPC_ARCHIVE_BALANCES_WINDOW=200
 # ETHEREUM_JSONRPC_HTTP_HEADERS=
 # ETHEREUM_JSONRPC_WAIT_PER_TIMEOUT=
+# ETHEREUM_JSONRPC_GETH_TRACE_BY_BLOCK=
 IPC_PATH=
 NETWORK_PATH=/
 BLOCKSCOUT_HOST=