Re-implement Geth JS internal transaction tracer in Elixir

Currently in Geth variant raw traces are converted into internal_transactions
using a custom JS tracer in [1]. cpp-ethereum nodes don't support custom JS
tracers and return raw traces regardless of provided parameters.

This PR adds an Elixir implementation of the tracer, which is utilized in
case raw traces are detected in the response from the node. It was
cross-checked for correctness with Geth builtin `callTracer`[2].
In the process some bugs were discovered in the our JS tracer, which will
be fixed with a separate PR.

Though, unlike JS tracer, we don't have access to EVM internal state during
tracing, so additional requests to node are required: transactions and receipts
are fetched before tracing, created contract codes and pre-selfdestruct
balances are fetched after.

[1] priv/js/ethereum_jsonrpc/geth/debug_traceTransaction/tracer.js
[2] https://github.com/ethereum/go-ethereum/blob/master/eth/tracers/internal/tracers/call_tracer.js

apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/geth.ex

@@ -3,9 +3,10 @@ defmodule EthereumJSONRPC.Geth do
   Ethereum JSONRPC methods that are only supported by [Geth](https://github.com/ethereum/go-ethereum/wiki/geth).
   """
 
-  import EthereumJSONRPC, only: [id_to_params: 1, json_rpc: 2, request: 1]
+  import EthereumJSONRPC, only: [id_to_params: 1, integer_to_quantity: 1, json_rpc: 2, request: 1]
 
-  alias EthereumJSONRPC.Geth.Calls
+  alias EthereumJSONRPC.{FetchedBalance, FetchedCode}
+  alias EthereumJSONRPC.Geth.{Calls, Tracer}
 
   @behaviour EthereumJSONRPC.Variant
 
@@ -28,7 +29,11 @@ defmodule EthereumJSONRPC.Geth do
            id_to_params
            |> debug_trace_transaction_requests()
            |> json_rpc(json_rpc_named_arguments) do
-      debug_trace_transaction_responses_to_internal_transactions_params(responses, id_to_params)
+      debug_trace_transaction_responses_to_internal_transactions_params(
+        responses,
+        id_to_params,
+        json_rpc_named_arguments
+      )
     end
   end
 
@@ -62,13 +67,88 @@ defmodule EthereumJSONRPC.Geth do
     request(%{id: id, method: "debug_traceTransaction", params: [hash_data, %{tracer: @tracer}]})
   end
 
-  defp debug_trace_transaction_responses_to_internal_transactions_params(responses, id_to_params)
+  defp debug_trace_transaction_responses_to_internal_transactions_params(
+         [%{result: %{"structLogs" => _}} | _] = responses,
+         id_to_params,
+         json_rpc_named_arguments
+       )
+       when is_map(id_to_params) do
+    with {:ok, receipts} <-
+           id_to_params
+           |> Enum.map(fn {id, %{hash_data: hash_data}} ->
+             request(%{id: id, method: "eth_getTransactionReceipt", params: [hash_data]})
+           end)
+           |> json_rpc(json_rpc_named_arguments),
+         {:ok, txs} <-
+           id_to_params
+           |> Enum.map(fn {id, %{hash_data: hash_data}} ->
+             request(%{id: id, method: "eth_getTransactionByHash", params: [hash_data]})
+           end)
+           |> json_rpc(json_rpc_named_arguments) do
+      receipts_map = Enum.into(receipts, %{}, fn %{id: id, result: receipt} -> {id, receipt} end)
+      txs_map = Enum.into(txs, %{}, fn %{id: id, result: tx} -> {id, tx} end)
+
+      responses
+      |> Enum.map(fn %{id: id, result: %{"structLogs" => _} = result} ->
+        debug_trace_transaction_response_to_internal_transactions_params(
+          %{id: id, result: Tracer.replay(result, Map.fetch!(receipts_map, id), Map.fetch!(txs_map, id))},
+          id_to_params
+        )
+      end)
+      |> reduce_internal_transactions_params()
+      |> fetch_missing_data(json_rpc_named_arguments)
+    end
+  end
+
+  defp debug_trace_transaction_responses_to_internal_transactions_params(
+         responses,
+         id_to_params,
+         _json_rpc_named_arguments
+       )
        when is_list(responses) and is_map(id_to_params) do
     responses
     |> Enum.map(&debug_trace_transaction_response_to_internal_transactions_params(&1, id_to_params))
     |> reduce_internal_transactions_params()
   end
 
+  defp fetch_missing_data({:ok, transactions}, json_rpc_named_arguments) when is_list(transactions) do
+    id_to_params = id_to_params(transactions)
+
+    with {:ok, responses} <-
+           id_to_params
+           |> Enum.map(fn
+             {id, %{created_contract_address_hash: address, block_number: block_number}} ->
+               FetchedCode.request(%{id: id, block_quantity: integer_to_quantity(block_number), address: address})
+
+             {id, %{type: "selfdestruct", from: hash_data, block_number: block_number}} ->
+               FetchedBalance.request(%{id: id, block_quantity: integer_to_quantity(block_number), hash_data: hash_data})
+
+             _ ->
+               nil
+           end)
+           |> Enum.reject(&is_nil/1)
+           |> json_rpc(json_rpc_named_arguments) do
+      results = Enum.into(responses, %{}, fn %{id: id, result: result} -> {id, result} end)
+
+      transactions =
+        id_to_params
+        |> Enum.map(fn
+          {id, %{created_contract_address_hash: _} = transaction} ->
+            %{transaction | created_contract_code: Map.fetch!(results, id)}
+
+          {id, %{type: "selfdestruct"} = transaction} ->
+            %{transaction | value: Map.fetch!(results, id)}
+
+          {_, transaction} ->
+            transaction
+        end)
+
+      {:ok, transactions}
+    end
+  end
+
+  defp fetch_missing_data(result, _json_rpc_named_arguments), do: result
+
   defp debug_trace_transaction_response_to_internal_transactions_params(%{id: id, result: calls}, id_to_params)
        when is_map(id_to_params) do
     %{block_number: block_number, hash_data: transaction_hash, transaction_index: transaction_index} =

apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/geth/call.ex

@@ -328,9 +328,8 @@ defmodule EthereumJSONRPC.Geth.Call do
          "from" => from_address_hash,
          "to" => to_address_hash,
          "gas" => gas,
-         "gasUsed" => gas_used,
          "input" => input,
-         "output" => output,
+         "error" => error,
          "value" => value
        })
        when call_type in ~w(call callcode delegatecall) do
@@ -345,9 +344,8 @@ defmodule EthereumJSONRPC.Geth.Call do
       from_address_hash: from_address_hash,
       to_address_hash: to_address_hash,
       gas: gas,
-      gas_used: gas_used,
       input: input,
-      output: output,
+      error: error,
       value: value
     }
   end
@@ -363,8 +361,9 @@ defmodule EthereumJSONRPC.Geth.Call do
          "from" => from_address_hash,
          "to" => to_address_hash,
          "gas" => gas,
+         "gasUsed" => gas_used,
          "input" => input,
-         "error" => error,
+         "output" => output,
          "value" => value
        })
        when call_type in ~w(call callcode delegatecall) do
@@ -379,8 +378,9 @@ defmodule EthereumJSONRPC.Geth.Call do
       from_address_hash: from_address_hash,
       to_address_hash: to_address_hash,
       gas: gas,
+      gas_used: gas_used,
       input: input,
-      error: error,
+      output: output,
       value: value
     }
   end
@@ -425,13 +425,11 @@ defmodule EthereumJSONRPC.Geth.Call do
          "transactionHash" => transaction_hash,
          "index" => index,
          "traceAddress" => trace_address,
-         "type" => "create",
+         "type" => "create" = type,
          "from" => from_address_hash,
-         "createdContractAddressHash" => created_contract_address_hash,
+         "error" => error,
          "gas" => gas,
-         "gasUsed" => gas_used,
          "init" => init,
-         "createdContractCode" => created_contract_code,
          "value" => value
        }) do
     %{
@@ -440,13 +438,11 @@ defmodule EthereumJSONRPC.Geth.Call do
       transaction_hash: transaction_hash,
       index: index,
       trace_address: trace_address,
-      type: "create",
+      type: type,
       from_address_hash: from_address_hash,
       gas: gas,
-      gas_used: gas_used,
+      error: error,
-      created_contract_address_hash: created_contract_address_hash,
       init: init,
-      created_contract_code: created_contract_code,
       value: value
     }
   end
@@ -457,11 +453,13 @@ defmodule EthereumJSONRPC.Geth.Call do
          "transactionHash" => transaction_hash,
          "index" => index,
          "traceAddress" => trace_address,
-         "type" => "create" = type,
+         "type" => "create",
          "from" => from_address_hash,
-         "error" => error,
+         "createdContractAddressHash" => created_contract_address_hash,
          "gas" => gas,
+         "gasUsed" => gas_used,
          "init" => init,
+         "createdContractCode" => created_contract_code,
          "value" => value
        }) do
     %{
@@ -470,11 +468,13 @@ defmodule EthereumJSONRPC.Geth.Call do
       transaction_hash: transaction_hash,
       index: index,
       trace_address: trace_address,
-      type: type,
+      type: "create",
       from_address_hash: from_address_hash,
       gas: gas,
-      error: error,
+      gas_used: gas_used,
+      created_contract_address_hash: created_contract_address_hash,
       init: init,
+      created_contract_code: created_contract_code,
       value: value
     }
   end

apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/geth/tracer.ex

@@ -0,0 +1,296 @@
+defmodule EthereumJSONRPC.Geth.Tracer do
+  @moduledoc """
+  Elixir implementation of a custom tracer (`priv/js/ethereum_jsonrpc/geth/debug_traceTransaction/tracer.js`)
+  for variants that don't support specifying tracer in [debug_traceTransaction](https://github.com/ethereum/go-ethereum/wiki/Management-APIs#debug_tracetransaction) calls.
+  """
+
+  import EthereumJSONRPC, only: [integer_to_quantity: 1, quantity_to_integer: 1]
+
+  def replay(%{"structLogs" => logs} = result, receipt, tx) when is_list(logs) do
+    %{"from" => from, "to" => to, "contractAddress" => contract_address, "gasUsed" => gas_used} = receipt
+    %{"value" => value, "input" => input, "gas" => gas} = tx
+
+    top =
+      to
+      |> if do
+        %{
+          "type" => "call",
+          "callType" => "call",
+          "to" => to,
+          "input" => input,
+          "output" => Map.get(result, "return", "0x" <> Map.get(result, "returnValue", ""))
+        }
+      else
+        %{
+          "type" => "create",
+          "init" => input,
+          "createdContractAddressHash" => contract_address,
+          "createdContractCode" => "0x"
+        }
+      end
+      |> Map.merge(%{
+        "from" => from,
+        "traceAddress" => [],
+        "value" => value,
+        "gas" => 0,
+        "gasUsed" => quantity_to_integer(gas_used) - quantity_to_integer(gas)
+      })
+
+    ctx = %{
+      depth: 1,
+      stack: [top],
+      trace_address: [0],
+      calls: [[]]
+    }
+
+    logs
+    |> Enum.reduce(ctx, &step/2)
+    |> finalize()
+  end
+
+  defp step(%{"error" => _}, %{stack: [%{"error" => _} | _]} = ctx), do: ctx
+
+  defp step(
+         %{"error" => _} = log,
+         %{
+           depth: stack_depth,
+           stack: [call | stack],
+           trace_address: [_, trace_index | trace_address],
+           calls: [subsubcalls, subcalls | calls]
+         } = ctx
+       ) do
+    call = process_return(log, Map.put(call, "error", "error"))
+
+    subsubcalls =
+      subsubcalls
+      |> Enum.reverse()
+      |> Enum.map(fn
+        subcalls when is_list(subcalls) -> subcalls
+        subcall when is_map(subcall) -> %{subcall | "from" => call["createdContractAddressHash"] || call["to"]}
+      end)
+
+    %{
+      ctx
+      | depth: stack_depth - 1,
+        stack: stack,
+        trace_address: [trace_index + 1 | trace_address],
+        calls: [[subsubcalls, call | subcalls] | calls]
+    }
+  end
+
+  defp step(
+         %{"gas" => log_gas} = log,
+         %{stack: [%{"type" => "create", "gas" => 0, "gasUsed" => 0} = call | stack]} = ctx
+       ) do
+    step(log, %{ctx | stack: [%{call | "gas" => log_gas, "gasUsed" => log_gas} | stack]})
+  end
+
+  defp step(%{"gas" => log_gas} = log, %{stack: [%{"type" => "create", "gas" => 0} = call | stack]} = ctx) do
+    step(log, %{ctx | stack: [%{call | "gas" => log_gas} | stack]})
+  end
+
+  defp step(%{"gas" => log_gas} = log, %{stack: [%{"gas" => 0, "gasUsed" => gas_used} = call | stack]} = ctx) do
+    step(log, %{ctx | stack: [%{call | "gas" => log_gas, "gasUsed" => gas_used + log_gas} | stack]})
+  end
+
+  defp step(
+         %{"depth" => log_depth, "gas" => log_gas} = log,
+         %{
+           depth: stack_depth,
+           stack: [call | stack],
+           trace_address: [_, trace_index | trace_address],
+           calls: [subsubcalls, subcalls | calls]
+         } = ctx
+       )
+       when log_depth == stack_depth - 1 do
+    call = process_return(log, %{call | "gasUsed" => call["gasUsed"] - log_gas})
+
+    subsubcalls =
+      subsubcalls
+      |> Enum.reverse()
+      |> Enum.map(fn
+        subcalls when is_list(subcalls) -> subcalls
+        subcall when is_map(subcall) -> %{subcall | "from" => call["createdContractAddressHash"] || call["to"]}
+      end)
+
+    step(log, %{
+      ctx
+      | depth: stack_depth - 1,
+        stack: stack,
+        trace_address: [trace_index + 1 | trace_address],
+        calls: [[subsubcalls, call | subcalls] | calls]
+    })
+  end
+
+  defp step(%{"op" => "CREATE"} = log, ctx), do: create_op(log, ctx)
+  defp step(%{"op" => "SELFDESTRUCT"} = log, ctx), do: self_destruct_op(log, ctx)
+  defp step(%{"op" => "CALL"} = log, ctx), do: call_op(log, "call", ctx)
+  defp step(%{"op" => "CALLCODE"} = log, ctx), do: call_op(log, "callcode", ctx)
+  defp step(%{"op" => "DELEGATECALL"} = log, ctx), do: call_op(log, "delegatecall", ctx)
+  defp step(%{"op" => "STATICCALL"} = log, ctx), do: call_op(log, "staticcall", ctx)
+  defp step(%{"op" => "REVERT"}, ctx), do: revert_op(ctx)
+  defp step(_, ctx), do: ctx
+
+  defp process_return(%{"stack" => log_stack}, %{"type" => "create"} = call) do
+    [ret | _] = Enum.reverse(log_stack)
+
+    case quantity_to_integer(ret) do
+      0 -> Map.put(call, "error", call["error"] || "internal failure")
+      _ -> %{call | "createdContractAddressHash" => "0x" <> String.slice(ret, 24, 40)}
+    end
+  end
+
+  defp process_return(
+         %{"stack" => log_stack, "memory" => log_memory},
+         %{"outputOffset" => out_off, "outputLength" => out_len} = call
+       ) do
+    [ret | _] = Enum.reverse(log_stack)
+
+    ret
+    |> quantity_to_integer()
+    |> case do
+      0 ->
+        Map.put(call, "error", call["error"] || "internal failure")
+
+      _ ->
+        output =
+          log_memory
+          |> IO.iodata_to_binary()
+          |> String.slice(out_off, out_len)
+
+        %{call | "output" => "0x" <> output}
+    end
+    |> Map.drop(["outputOffset", "outputLength"])
+  end
+
+  defp create_op(
+         %{"stack" => log_stack, "memory" => log_memory, "gas" => log_gas, "gasCost" => log_gas_cost},
+         %{depth: stack_depth, stack: stack, trace_address: trace_address, calls: calls} = ctx
+       ) do
+    [value, input_offset, input_length | _] = Enum.reverse(log_stack)
+
+    init =
+      log_memory
+      |> IO.iodata_to_binary()
+      |> String.slice(quantity_to_integer("0x" <> input_offset) * 2, quantity_to_integer("0x" <> input_length) * 2)
+
+    call = %{
+      "type" => "create",
+      "from" => nil,
+      "traceAddress" => Enum.reverse(trace_address),
+      "init" => "0x" <> init,
+      "gas" => 0,
+      "gasUsed" => log_gas - log_gas_cost,
+      "value" => "0x" <> value,
+      "createdContractAddressHash" => nil,
+      "createdContractCode" => "0x"
+    }
+
+    %{
+      ctx
+      | depth: stack_depth + 1,
+        stack: [call | stack],
+        trace_address: [0 | trace_address],
+        calls: [[] | calls]
+    }
+  end
+
+  defp self_destruct_op(
+         %{"stack" => log_stack, "gas" => log_gas, "gasCost" => log_gas_cost},
+         %{trace_address: [trace_index | trace_address], calls: [subcalls | calls]} = ctx
+       ) do
+    [to | _] = Enum.reverse(log_stack)
+
+    if quantity_to_integer(to) in 1..8 do
+      ctx
+    else
+      call = %{
+        "type" => "selfdestruct",
+        "from" => nil,
+        "to" => "0x" <> String.slice(to, 24, 40),
+        "traceAddress" => Enum.reverse([trace_index | trace_address]),
+        "gas" => log_gas,
+        "gasUsed" => log_gas_cost,
+        "value" => "0x0"
+      }
+
+      %{ctx | trace_address: [trace_index + 1 | trace_address], calls: [[call | subcalls] | calls]}
+    end
+  end
+
+  defp call_op(
+         %{"stack" => log_stack, "memory" => log_memory, "gas" => log_gas, "gasCost" => log_gas_cost},
+         call_type,
+         %{
+           depth: stack_depth,
+           stack: [%{"value" => parent_value} = parent | stack],
+           trace_address: trace_address,
+           calls: calls
+         } = ctx
+       ) do
+    [_, to | log_stack] = Enum.reverse(log_stack)
+
+    {value, [input_offset, input_length, output_offset, output_length | _]} =
+      case call_type do
+        "delegatecall" ->
+          {parent_value, log_stack}
+
+        "staticcall" ->
+          {"0x0", log_stack}
+
+        _ ->
+          [value | rest] = log_stack
+          {"0x" <> value, rest}
+      end
+
+    input =
+      log_memory
+      |> IO.iodata_to_binary()
+      |> String.slice(quantity_to_integer("0x" <> input_offset) * 2, quantity_to_integer("0x" <> input_length) * 2)
+
+    call = %{
+      "type" => "call",
+      "callType" => call_type,
+      "from" => nil,
+      "to" => "0x" <> String.slice(to, 24, 40),
+      "traceAddress" => Enum.reverse(trace_address),
+      "input" => "0x" <> input,
+      "output" => "0x",
+      "outputOffset" => quantity_to_integer("0x" <> output_offset) * 2,
+      "outputLength" => quantity_to_integer("0x" <> output_length) * 2,
+      "gas" => 0,
+      "gasUsed" => log_gas - log_gas_cost,
+      "value" => value
+    }
+
+    %{
+      ctx
+      | depth: stack_depth + 1,
+        stack: [call, parent | stack],
+        trace_address: [0 | trace_address],
+        calls: [[] | calls]
+    }
+  end
+
+  defp revert_op(%{stack: [last | stack]} = ctx) do
+    %{ctx | stack: [Map.put(last, "error", "execution reverted") | stack]}
+  end
+
+  defp finalize(%{stack: [top], calls: [calls]}) do
+    calls =
+      Enum.map(calls, fn
+        subcalls when is_list(subcalls) -> subcalls
+        subcall when is_map(subcall) -> %{subcall | "from" => top["createdContractAddressHash"] || top["to"]}
+      end)
+
+    [top | Enum.reverse(calls)]
+    |> List.flatten()
+    |> Enum.map(fn
+      %{"gas" => gas, "gasUsed" => gas_used} = call when gas_used < 0 ->
+        %{call | "gas" => integer_to_quantity(gas - gas_used), "gasUsed" => "0x0"}
+
+      %{"gas" => gas, "gasUsed" => gas_used} = call ->
+        %{call | "gas" => integer_to_quantity(gas), "gasUsed" => integer_to_quantity(gas_used)}
+    end)
+  end
+end