From 1aeea2b9cad20ad0b2a79dad2c8e26c1eb1658c3 Mon Sep 17 00:00:00 2001
From: goodsoft <goodsoft@goodsoft.lv>
Date: Mon, 29 Apr 2019 17:40:20 +0300
Subject: [PATCH] Re-implement Geth JS internal transaction tracer in Elixir

Currently in Geth variant raw traces are converted into internal_transactions
using a custom JS tracer in [1]. cpp-ethereum nodes don't support custom JS
tracers and return raw traces regardless of provided parameters.

This PR adds an Elixir implementation of the tracer, which is utilized in
case raw traces are detected in the response from the node. It was
cross-checked for correctness with Geth builtin `callTracer`[2].
In the process some bugs were discovered in the our JS tracer, which will
be fixed with a separate PR.

Though, unlike JS tracer, we don't have access to EVM internal state during
tracing, so additional requests to node are required: transactions and receipts
are fetched before tracing, created contract codes and pre-selfdestruct
balances are fetched after.

[1] priv/js/ethereum_jsonrpc/geth/debug_traceTransaction/tracer.js
[2] https://github.com/ethereum/go-ethereum/blob/master/eth/tracers/internal/tracers/call_tracer.js
---
 .../lib/ethereum_jsonrpc/geth.ex              |  88 +++++-
 .../lib/ethereum_jsonrpc/geth/call.ex         |  36 +--
 .../lib/ethereum_jsonrpc/geth/tracer.ex       | 296 ++++++++++++++++++
 3 files changed, 398 insertions(+), 22 deletions(-)
 create mode 100644 apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/geth/tracer.ex

diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/geth.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/geth.ex
index b4a4e7c922..ab725c9c73 100644
--- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/geth.ex
+++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/geth.ex
@@ -3,9 +3,10 @@ defmodule EthereumJSONRPC.Geth do
   Ethereum JSONRPC methods that are only supported by [Geth](https://github.com/ethereum/go-ethereum/wiki/geth).
   """
 
-  import EthereumJSONRPC, only: [id_to_params: 1, json_rpc: 2, request: 1]
+  import EthereumJSONRPC, only: [id_to_params: 1, integer_to_quantity: 1, json_rpc: 2, request: 1]
 
-  alias EthereumJSONRPC.Geth.Calls
+  alias EthereumJSONRPC.{FetchedBalance, FetchedCode}
+  alias EthereumJSONRPC.Geth.{Calls, Tracer}
 
   @behaviour EthereumJSONRPC.Variant
 
@@ -28,7 +29,11 @@ defmodule EthereumJSONRPC.Geth do
            id_to_params
            |> debug_trace_transaction_requests()
            |> json_rpc(json_rpc_named_arguments) do
-      debug_trace_transaction_responses_to_internal_transactions_params(responses, id_to_params)
+      debug_trace_transaction_responses_to_internal_transactions_params(
+        responses,
+        id_to_params,
+        json_rpc_named_arguments
+      )
     end
   end
 
@@ -62,13 +67,88 @@ defmodule EthereumJSONRPC.Geth do
     request(%{id: id, method: "debug_traceTransaction", params: [hash_data, %{tracer: @tracer}]})
   end
 
-  defp debug_trace_transaction_responses_to_internal_transactions_params(responses, id_to_params)
+  defp debug_trace_transaction_responses_to_internal_transactions_params(
+         [%{result: %{"structLogs" => _}} | _] = responses,
+         id_to_params,
+         json_rpc_named_arguments
+       )
+       when is_map(id_to_params) do
+    with {:ok, receipts} <-
+           id_to_params
+           |> Enum.map(fn {id, %{hash_data: hash_data}} ->
+             request(%{id: id, method: "eth_getTransactionReceipt", params: [hash_data]})
+           end)
+           |> json_rpc(json_rpc_named_arguments),
+         {:ok, txs} <-
+           id_to_params
+           |> Enum.map(fn {id, %{hash_data: hash_data}} ->
+             request(%{id: id, method: "eth_getTransactionByHash", params: [hash_data]})
+           end)
+           |> json_rpc(json_rpc_named_arguments) do
+      receipts_map = Enum.into(receipts, %{}, fn %{id: id, result: receipt} -> {id, receipt} end)
+      txs_map = Enum.into(txs, %{}, fn %{id: id, result: tx} -> {id, tx} end)
+
+      responses
+      |> Enum.map(fn %{id: id, result: %{"structLogs" => _} = result} ->
+        debug_trace_transaction_response_to_internal_transactions_params(
+          %{id: id, result: Tracer.replay(result, Map.fetch!(receipts_map, id), Map.fetch!(txs_map, id))},
+          id_to_params
+        )
+      end)
+      |> reduce_internal_transactions_params()
+      |> fetch_missing_data(json_rpc_named_arguments)
+    end
+  end
+
+  defp debug_trace_transaction_responses_to_internal_transactions_params(
+         responses,
+         id_to_params,
+         _json_rpc_named_arguments
+       )
        when is_list(responses) and is_map(id_to_params) do
     responses
     |> Enum.map(&debug_trace_transaction_response_to_internal_transactions_params(&1, id_to_params))
     |> reduce_internal_transactions_params()
   end
 
+  defp fetch_missing_data({:ok, transactions}, json_rpc_named_arguments) when is_list(transactions) do
+    id_to_params = id_to_params(transactions)
+
+    with {:ok, responses} <-
+           id_to_params
+           |> Enum.map(fn
+             {id, %{created_contract_address_hash: address, block_number: block_number}} ->
+               FetchedCode.request(%{id: id, block_quantity: integer_to_quantity(block_number), address: address})
+
+             {id, %{type: "selfdestruct", from: hash_data, block_number: block_number}} ->
+               FetchedBalance.request(%{id: id, block_quantity: integer_to_quantity(block_number), hash_data: hash_data})
+
+             _ ->
+               nil
+           end)
+           |> Enum.reject(&is_nil/1)
+           |> json_rpc(json_rpc_named_arguments) do
+      results = Enum.into(responses, %{}, fn %{id: id, result: result} -> {id, result} end)
+
+      transactions =
+        id_to_params
+        |> Enum.map(fn
+          {id, %{created_contract_address_hash: _} = transaction} ->
+            %{transaction | created_contract_code: Map.fetch!(results, id)}
+
+          {id, %{type: "selfdestruct"} = transaction} ->
+            %{transaction | value: Map.fetch!(results, id)}
+
+          {_, transaction} ->
+            transaction
+        end)
+
+      {:ok, transactions}
+    end
+  end
+
+  defp fetch_missing_data(result, _json_rpc_named_arguments), do: result
+
   defp debug_trace_transaction_response_to_internal_transactions_params(%{id: id, result: calls}, id_to_params)
        when is_map(id_to_params) do
     %{block_number: block_number, hash_data: transaction_hash, transaction_index: transaction_index} =
diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/geth/call.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/geth/call.ex
index 88a32c704f..434cc7b34d 100644
--- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/geth/call.ex
+++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/geth/call.ex
@@ -328,9 +328,8 @@ defmodule EthereumJSONRPC.Geth.Call do
          "from" => from_address_hash,
          "to" => to_address_hash,
          "gas" => gas,
-         "gasUsed" => gas_used,
          "input" => input,
-         "output" => output,
+         "error" => error,
          "value" => value
        })
        when call_type in ~w(call callcode delegatecall) do
@@ -345,9 +344,8 @@ defmodule EthereumJSONRPC.Geth.Call do
       from_address_hash: from_address_hash,
       to_address_hash: to_address_hash,
       gas: gas,
-      gas_used: gas_used,
       input: input,
-      output: output,
+      error: error,
       value: value
     }
   end
@@ -363,8 +361,9 @@ defmodule EthereumJSONRPC.Geth.Call do
          "from" => from_address_hash,
          "to" => to_address_hash,
          "gas" => gas,
+         "gasUsed" => gas_used,
          "input" => input,
-         "error" => error,
+         "output" => output,
          "value" => value
        })
        when call_type in ~w(call callcode delegatecall) do
@@ -379,8 +378,9 @@ defmodule EthereumJSONRPC.Geth.Call do
       from_address_hash: from_address_hash,
       to_address_hash: to_address_hash,
       gas: gas,
+      gas_used: gas_used,
       input: input,
-      error: error,
+      output: output,
       value: value
     }
   end
@@ -425,13 +425,11 @@ defmodule EthereumJSONRPC.Geth.Call do
          "transactionHash" => transaction_hash,
          "index" => index,
          "traceAddress" => trace_address,
-         "type" => "create",
+         "type" => "create" = type,
          "from" => from_address_hash,
-         "createdContractAddressHash" => created_contract_address_hash,
+         "error" => error,
          "gas" => gas,
-         "gasUsed" => gas_used,
          "init" => init,
-         "createdContractCode" => created_contract_code,
          "value" => value
        }) do
     %{
@@ -440,13 +438,11 @@ defmodule EthereumJSONRPC.Geth.Call do
       transaction_hash: transaction_hash,
       index: index,
       trace_address: trace_address,
-      type: "create",
+      type: type,
       from_address_hash: from_address_hash,
       gas: gas,
-      gas_used: gas_used,
-      created_contract_address_hash: created_contract_address_hash,
+      error: error,
       init: init,
-      created_contract_code: created_contract_code,
       value: value
     }
   end
@@ -457,11 +453,13 @@ defmodule EthereumJSONRPC.Geth.Call do
          "transactionHash" => transaction_hash,
          "index" => index,
          "traceAddress" => trace_address,
-         "type" => "create" = type,
+         "type" => "create",
          "from" => from_address_hash,
-         "error" => error,
+         "createdContractAddressHash" => created_contract_address_hash,
          "gas" => gas,
+         "gasUsed" => gas_used,
          "init" => init,
+         "createdContractCode" => created_contract_code,
          "value" => value
        }) do
     %{
@@ -470,11 +468,13 @@ defmodule EthereumJSONRPC.Geth.Call do
       transaction_hash: transaction_hash,
       index: index,
       trace_address: trace_address,
-      type: type,
+      type: "create",
       from_address_hash: from_address_hash,
       gas: gas,
-      error: error,
+      gas_used: gas_used,
+      created_contract_address_hash: created_contract_address_hash,
       init: init,
+      created_contract_code: created_contract_code,
       value: value
     }
   end
diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/geth/tracer.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/geth/tracer.ex
new file mode 100644
index 0000000000..98fe2c6719
--- /dev/null
+++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/geth/tracer.ex
@@ -0,0 +1,296 @@
+defmodule EthereumJSONRPC.Geth.Tracer do
+  @moduledoc """
+  Elixir implementation of a custom tracer (`priv/js/ethereum_jsonrpc/geth/debug_traceTransaction/tracer.js`)
+  for variants that don't support specifying tracer in [debug_traceTransaction](https://github.com/ethereum/go-ethereum/wiki/Management-APIs#debug_tracetransaction) calls.
+  """
+
+  import EthereumJSONRPC, only: [integer_to_quantity: 1, quantity_to_integer: 1]
+
+  def replay(%{"structLogs" => logs} = result, receipt, tx) when is_list(logs) do
+    %{"from" => from, "to" => to, "contractAddress" => contract_address, "gasUsed" => gas_used} = receipt
+    %{"value" => value, "input" => input, "gas" => gas} = tx
+
+    top =
+      to
+      |> if do
+        %{
+          "type" => "call",
+          "callType" => "call",
+          "to" => to,
+          "input" => input,
+          "output" => Map.get(result, "return", "0x" <> Map.get(result, "returnValue", ""))
+        }
+      else
+        %{
+          "type" => "create",
+          "init" => input,
+          "createdContractAddressHash" => contract_address,
+          "createdContractCode" => "0x"
+        }
+      end
+      |> Map.merge(%{
+        "from" => from,
+        "traceAddress" => [],
+        "value" => value,
+        "gas" => 0,
+        "gasUsed" => quantity_to_integer(gas_used) - quantity_to_integer(gas)
+      })
+
+    ctx = %{
+      depth: 1,
+      stack: [top],
+      trace_address: [0],
+      calls: [[]]
+    }
+
+    logs
+    |> Enum.reduce(ctx, &step/2)
+    |> finalize()
+  end
+
+  defp step(%{"error" => _}, %{stack: [%{"error" => _} | _]} = ctx), do: ctx
+
+  defp step(
+         %{"error" => _} = log,
+         %{
+           depth: stack_depth,
+           stack: [call | stack],
+           trace_address: [_, trace_index | trace_address],
+           calls: [subsubcalls, subcalls | calls]
+         } = ctx
+       ) do
+    call = process_return(log, Map.put(call, "error", "error"))
+
+    subsubcalls =
+      subsubcalls
+      |> Enum.reverse()
+      |> Enum.map(fn
+        subcalls when is_list(subcalls) -> subcalls
+        subcall when is_map(subcall) -> %{subcall | "from" => call["createdContractAddressHash"] || call["to"]}
+      end)
+
+    %{
+      ctx
+      | depth: stack_depth - 1,
+        stack: stack,
+        trace_address: [trace_index + 1 | trace_address],
+        calls: [[subsubcalls, call | subcalls] | calls]
+    }
+  end
+
+  defp step(
+         %{"gas" => log_gas} = log,
+         %{stack: [%{"type" => "create", "gas" => 0, "gasUsed" => 0} = call | stack]} = ctx
+       ) do
+    step(log, %{ctx | stack: [%{call | "gas" => log_gas, "gasUsed" => log_gas} | stack]})
+  end
+
+  defp step(%{"gas" => log_gas} = log, %{stack: [%{"type" => "create", "gas" => 0} = call | stack]} = ctx) do
+    step(log, %{ctx | stack: [%{call | "gas" => log_gas} | stack]})
+  end
+
+  defp step(%{"gas" => log_gas} = log, %{stack: [%{"gas" => 0, "gasUsed" => gas_used} = call | stack]} = ctx) do
+    step(log, %{ctx | stack: [%{call | "gas" => log_gas, "gasUsed" => gas_used + log_gas} | stack]})
+  end
+
+  defp step(
+         %{"depth" => log_depth, "gas" => log_gas} = log,
+         %{
+           depth: stack_depth,
+           stack: [call | stack],
+           trace_address: [_, trace_index | trace_address],
+           calls: [subsubcalls, subcalls | calls]
+         } = ctx
+       )
+       when log_depth == stack_depth - 1 do
+    call = process_return(log, %{call | "gasUsed" => call["gasUsed"] - log_gas})
+
+    subsubcalls =
+      subsubcalls
+      |> Enum.reverse()
+      |> Enum.map(fn
+        subcalls when is_list(subcalls) -> subcalls
+        subcall when is_map(subcall) -> %{subcall | "from" => call["createdContractAddressHash"] || call["to"]}
+      end)
+
+    step(log, %{
+      ctx
+      | depth: stack_depth - 1,
+        stack: stack,
+        trace_address: [trace_index + 1 | trace_address],
+        calls: [[subsubcalls, call | subcalls] | calls]
+    })
+  end
+
+  defp step(%{"op" => "CREATE"} = log, ctx), do: create_op(log, ctx)
+  defp step(%{"op" => "SELFDESTRUCT"} = log, ctx), do: self_destruct_op(log, ctx)
+  defp step(%{"op" => "CALL"} = log, ctx), do: call_op(log, "call", ctx)
+  defp step(%{"op" => "CALLCODE"} = log, ctx), do: call_op(log, "callcode", ctx)
+  defp step(%{"op" => "DELEGATECALL"} = log, ctx), do: call_op(log, "delegatecall", ctx)
+  defp step(%{"op" => "STATICCALL"} = log, ctx), do: call_op(log, "staticcall", ctx)
+  defp step(%{"op" => "REVERT"}, ctx), do: revert_op(ctx)
+  defp step(_, ctx), do: ctx
+
+  defp process_return(%{"stack" => log_stack}, %{"type" => "create"} = call) do
+    [ret | _] = Enum.reverse(log_stack)
+
+    case quantity_to_integer(ret) do
+      0 -> Map.put(call, "error", call["error"] || "internal failure")
+      _ -> %{call | "createdContractAddressHash" => "0x" <> String.slice(ret, 24, 40)}
+    end
+  end
+
+  defp process_return(
+         %{"stack" => log_stack, "memory" => log_memory},
+         %{"outputOffset" => out_off, "outputLength" => out_len} = call
+       ) do
+    [ret | _] = Enum.reverse(log_stack)
+
+    ret
+    |> quantity_to_integer()
+    |> case do
+      0 ->
+        Map.put(call, "error", call["error"] || "internal failure")
+
+      _ ->
+        output =
+          log_memory
+          |> IO.iodata_to_binary()
+          |> String.slice(out_off, out_len)
+
+        %{call | "output" => "0x" <> output}
+    end
+    |> Map.drop(["outputOffset", "outputLength"])
+  end
+
+  defp create_op(
+         %{"stack" => log_stack, "memory" => log_memory, "gas" => log_gas, "gasCost" => log_gas_cost},
+         %{depth: stack_depth, stack: stack, trace_address: trace_address, calls: calls} = ctx
+       ) do
+    [value, input_offset, input_length | _] = Enum.reverse(log_stack)
+
+    init =
+      log_memory
+      |> IO.iodata_to_binary()
+      |> String.slice(quantity_to_integer("0x" <> input_offset) * 2, quantity_to_integer("0x" <> input_length) * 2)
+
+    call = %{
+      "type" => "create",
+      "from" => nil,
+      "traceAddress" => Enum.reverse(trace_address),
+      "init" => "0x" <> init,
+      "gas" => 0,
+      "gasUsed" => log_gas - log_gas_cost,
+      "value" => "0x" <> value,
+      "createdContractAddressHash" => nil,
+      "createdContractCode" => "0x"
+    }
+
+    %{
+      ctx
+      | depth: stack_depth + 1,
+        stack: [call | stack],
+        trace_address: [0 | trace_address],
+        calls: [[] | calls]
+    }
+  end
+
+  defp self_destruct_op(
+         %{"stack" => log_stack, "gas" => log_gas, "gasCost" => log_gas_cost},
+         %{trace_address: [trace_index | trace_address], calls: [subcalls | calls]} = ctx
+       ) do
+    [to | _] = Enum.reverse(log_stack)
+
+    if quantity_to_integer(to) in 1..8 do
+      ctx
+    else
+      call = %{
+        "type" => "selfdestruct",
+        "from" => nil,
+        "to" => "0x" <> String.slice(to, 24, 40),
+        "traceAddress" => Enum.reverse([trace_index | trace_address]),
+        "gas" => log_gas,
+        "gasUsed" => log_gas_cost,
+        "value" => "0x0"
+      }
+
+      %{ctx | trace_address: [trace_index + 1 | trace_address], calls: [[call | subcalls] | calls]}
+    end
+  end
+
+  defp call_op(
+         %{"stack" => log_stack, "memory" => log_memory, "gas" => log_gas, "gasCost" => log_gas_cost},
+         call_type,
+         %{
+           depth: stack_depth,
+           stack: [%{"value" => parent_value} = parent | stack],
+           trace_address: trace_address,
+           calls: calls
+         } = ctx
+       ) do
+    [_, to | log_stack] = Enum.reverse(log_stack)
+
+    {value, [input_offset, input_length, output_offset, output_length | _]} =
+      case call_type do
+        "delegatecall" ->
+          {parent_value, log_stack}
+
+        "staticcall" ->
+          {"0x0", log_stack}
+
+        _ ->
+          [value | rest] = log_stack
+          {"0x" <> value, rest}
+      end
+
+    input =
+      log_memory
+      |> IO.iodata_to_binary()
+      |> String.slice(quantity_to_integer("0x" <> input_offset) * 2, quantity_to_integer("0x" <> input_length) * 2)
+
+    call = %{
+      "type" => "call",
+      "callType" => call_type,
+      "from" => nil,
+      "to" => "0x" <> String.slice(to, 24, 40),
+      "traceAddress" => Enum.reverse(trace_address),
+      "input" => "0x" <> input,
+      "output" => "0x",
+      "outputOffset" => quantity_to_integer("0x" <> output_offset) * 2,
+      "outputLength" => quantity_to_integer("0x" <> output_length) * 2,
+      "gas" => 0,
+      "gasUsed" => log_gas - log_gas_cost,
+      "value" => value
+    }
+
+    %{
+      ctx
+      | depth: stack_depth + 1,
+        stack: [call, parent | stack],
+        trace_address: [0 | trace_address],
+        calls: [[] | calls]
+    }
+  end
+
+  defp revert_op(%{stack: [last | stack]} = ctx) do
+    %{ctx | stack: [Map.put(last, "error", "execution reverted") | stack]}
+  end
+
+  defp finalize(%{stack: [top], calls: [calls]}) do
+    calls =
+      Enum.map(calls, fn
+        subcalls when is_list(subcalls) -> subcalls
+        subcall when is_map(subcall) -> %{subcall | "from" => top["createdContractAddressHash"] || top["to"]}
+      end)
+
+    [top | Enum.reverse(calls)]
+    |> List.flatten()
+    |> Enum.map(fn
+      %{"gas" => gas, "gasUsed" => gas_used} = call when gas_used < 0 ->
+        %{call | "gas" => integer_to_quantity(gas - gas_used), "gasUsed" => "0x0"}
+
+      %{"gas" => gas, "gasUsed" => gas_used} = call ->
+        %{call | "gas" => integer_to_quantity(gas), "gasUsed" => integer_to_quantity(gas_used)}
+    end)
+  end
+end