Add 'transaction_index' in the fetchers to save at 'internal_transaction'

Add 'transaction_index' in the fetchers to save at 'internal_transaction'
pull/928/head
Amanda Sposito 6 years ago
parent 2be80321a3
commit b86335cf34
  1. 22
      apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/parity.ex
  2. 63
      apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/parity/trace.ex
  3. 9
      apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/transaction.ex
  4. 3
      apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/transactions.ex
  5. 4
      apps/ethereum_jsonrpc/test/ethereum_jsonrpc/http/mox_test.exs
  6. 32
      apps/ethereum_jsonrpc/test/ethereum_jsonrpc/parity_test.exs
  7. 3
      apps/explorer/lib/explorer/chain/internal_transaction.ex
  8. 12
      apps/explorer/test/explorer/chain/import_test.exs
  9. 5
      apps/indexer/lib/indexer/block/catchup/fetcher.ex
  10. 4
      apps/indexer/lib/indexer/block/fetcher.ex
  11. 8
      apps/indexer/lib/indexer/block/realtime/fetcher.ex
  12. 10
      apps/indexer/lib/indexer/internal_transaction/fetcher.ex
  13. 16
      apps/indexer/test/indexer/internal_transaction/fetcher_test.exs

@ -142,13 +142,19 @@ defmodule EthereumJSONRPC.Parity do
defp trace_replay_transaction_response_to_traces(%{id: id, result: %{"trace" => traces}}, id_to_params)
when is_list(traces) and is_map(id_to_params) do
%{block_number: block_number, hash_data: transaction_hash} = Map.fetch!(id_to_params, id)
%{block_number: block_number, hash_data: transaction_hash, transaction_index: transaction_index} =
Map.fetch!(id_to_params, id)
annotated_traces =
traces
|> Stream.with_index()
|> Enum.map(fn {trace, index} ->
Map.merge(trace, %{"blockNumber" => block_number, "index" => index, "transactionHash" => transaction_hash})
Map.merge(trace, %{
"blockNumber" => block_number,
"index" => index,
"transactionIndex" => transaction_index,
"transactionHash" => transaction_hash
})
end)
{:ok, annotated_traces}
@ -156,9 +162,15 @@ defmodule EthereumJSONRPC.Parity do
defp trace_replay_transaction_response_to_traces(%{id: id, error: error}, id_to_params)
when is_map(id_to_params) do
%{block_number: block_number, hash_data: transaction_hash} = Map.fetch!(id_to_params, id)
annotated_error = Map.put(error, :data, %{"blockNumber" => block_number, "transactionHash" => transaction_hash})
%{block_number: block_number, hash_data: transaction_hash, transaction_index: transaction_index} =
Map.fetch!(id_to_params, id)
annotated_error =
Map.put(error, :data, %{
"blockNumber" => block_number,
"transactionIndex" => transaction_index,
"transactionHash" => transaction_hash
})
{:error, annotated_error}
end

@ -28,7 +28,8 @@ defmodule EthereumJSONRPC.Parity.Trace do
...> "subtraces" => 0,
...> "traceAddress" => [],
...> "transactionHash" => "0x3a3eb134e6792ce9403ea4188e5e79693de9e4c94e499db132be086400da79e6",
...> "type" => "create"
...> "type" => "create",
...> "transactionIndex" => 0
...> }
...> )
%{
@ -43,7 +44,8 @@ defmodule EthereumJSONRPC.Parity.Trace do
trace_address: [],
transaction_hash: "0x3a3eb134e6792ce9403ea4188e5e79693de9e4c94e499db132be086400da79e6",
type: "create",
value: 0
value: 0,
transaction_index: 0
}
A create can fail due to a Bad Instruction in the `init` that is meant to form the `code` of the contract
@ -62,7 +64,8 @@ defmodule EthereumJSONRPC.Parity.Trace do
...> "subtraces" => 0,
...> "traceAddress" => [],
...> "transactionHash" => "0x3c624bb4852fb5e35a8f45644cec7a486211f6ba89034768a2b763194f22f97d",
...> "type" => "create"
...> "type" => "create",
...> "transactionIndex" => 0
...> }
...> )
%{
@ -75,7 +78,8 @@ defmodule EthereumJSONRPC.Parity.Trace do
trace_address: [],
transaction_hash: "0x3c624bb4852fb5e35a8f45644cec7a486211f6ba89034768a2b763194f22f97d",
type: "create",
value: 0
value: 0,
transaction_index: 0
}
Call type traces are generated when a method is called. Calls are further divided by call type.
@ -99,7 +103,8 @@ defmodule EthereumJSONRPC.Parity.Trace do
...> "subtraces" => 0,
...> "traceAddress" => [],
...> "transactionHash" => "0x3a3eb134e6792ce9403ea4188e5e79693de9e4c94e499db132be086400da79e6",
...> "type" => "call"
...> "type" => "call",
...> "transactionIndex" => 0
...> }
...> )
%{
@ -114,7 +119,8 @@ defmodule EthereumJSONRPC.Parity.Trace do
trace_address: [],
transaction_hash: "0x3a3eb134e6792ce9403ea4188e5e79693de9e4c94e499db132be086400da79e6",
type: "call",
value: 0
value: 0,
transaction_index: 0
}
Calls can error and be reverted
@ -135,7 +141,8 @@ defmodule EthereumJSONRPC.Parity.Trace do
...> "subtraces" => 7,
...> "traceAddress" => [],
...> "transactionHash" => "0xcd7c15dbbc797722bef6e1d551edfd644fc7f4fb2ccd6a7947b2d1ade9ed140b",
...> "type" => "call"
...> "type" => "call",
...> "transactionIndex" => 0
...> }
...> )
%{
@ -149,7 +156,8 @@ defmodule EthereumJSONRPC.Parity.Trace do
trace_address: [],
transaction_hash: "0xcd7c15dbbc797722bef6e1d551edfd644fc7f4fb2ccd6a7947b2d1ade9ed140b",
type: "call",
value: 10000000000000000
value: 10000000000000000,
transaction_index: 0
}
Suicides transfer a `"balance"` from `"address"` to `"refundAddress"`. These suicide-unique fields can be mapped to
@ -174,7 +182,8 @@ defmodule EthereumJSONRPC.Parity.Trace do
...> "subtraces" => 0,
...> "traceAddress" => [0],
...> "transactionHash" => "0xb012b8c53498c669d87d85ed90f57385848b86d3f44ed14b2784ec685d6fda98",
...> "type" => "suicide"
...> "type" => "suicide",
...> "transactionIndex" => 0
...> }
...> )
%{
@ -185,7 +194,8 @@ defmodule EthereumJSONRPC.Parity.Trace do
trace_address: [0],
transaction_hash: "0xb012b8c53498c669d87d85ed90f57385848b86d3f44ed14b2784ec685d6fda98",
type: "suicide",
value: 0
value: 0,
transaction_index: 0
}
"""
@ -202,7 +212,8 @@ defmodule EthereumJSONRPC.Parity.Trace do
"blockNumber" => block_number,
"index" => index,
"traceAddress" => trace_address,
"transactionHash" => transaction_hash
"transactionHash" => transaction_hash,
"transactionIndex" => transaction_index
} = elixir
%{
@ -215,7 +226,8 @@ defmodule EthereumJSONRPC.Parity.Trace do
trace_address: trace_address,
transaction_hash: transaction_hash,
type: type,
value: value
value: value,
transaction_index: transaction_index
}
|> put_call_error_or_result(elixir)
end
@ -226,7 +238,8 @@ defmodule EthereumJSONRPC.Parity.Trace do
"blockNumber" => block_number,
"index" => index,
"traceAddress" => trace_address,
"transactionHash" => transaction_hash
"transactionHash" => transaction_hash,
"transactionIndex" => transaction_index
} = elixir
%{
@ -238,7 +251,8 @@ defmodule EthereumJSONRPC.Parity.Trace do
trace_address: trace_address,
transaction_hash: transaction_hash,
type: type,
value: value
value: value,
transaction_index: transaction_index
}
|> put_create_error_or_result(elixir)
end
@ -253,7 +267,8 @@ defmodule EthereumJSONRPC.Parity.Trace do
"blockNumber" => block_number,
"index" => index,
"traceAddress" => trace_address,
"transactionHash" => transaction_hash
"transactionHash" => transaction_hash,
"transactionIndex" => transaction_index
} = elixir
%{
@ -264,7 +279,8 @@ defmodule EthereumJSONRPC.Parity.Trace do
trace_address: trace_address,
transaction_hash: transaction_hash,
type: type,
value: value
value: value,
transaction_index: transaction_index
}
end
@ -289,6 +305,7 @@ defmodule EthereumJSONRPC.Parity.Trace do
...> "subtraces" => 0,
...> "traceAddress" => [],
...> "transactionHash" => "0x3a3eb134e6792ce9403ea4188e5e79693de9e4c94e499db132be086400da79e6",
...> "transactionIndex" => 0,
...> "type" => "create"
...> }
...> )
@ -309,6 +326,7 @@ defmodule EthereumJSONRPC.Parity.Trace do
"subtraces" => 0,
"traceAddress" => [],
"transactionHash" => "0x3a3eb134e6792ce9403ea4188e5e79693de9e4c94e499db132be086400da79e6",
"transactionIndex" => 0,
"type" => "create"
}
@ -330,10 +348,11 @@ defmodule EthereumJSONRPC.Parity.Trace do
...> },
...> "subtraces" => 0,
...> "traceAddress" => [],
...> "transactionIndex" => 0,
...> "type" => "create"
...> }
...> )
** (ArgumentError) Caller must `Map.put/2` `"blockNumber"`, `"index"`, and `"transactionHash"` in trace
** (ArgumentError) Caller must `Map.put/2` `"blockNumber"`, `"index"`, `"transactionHash"` and `"transactionIndex"` in trace
`"suicide"` `"type"` traces are different in that they have a `nil` `"result"`. This is because the `"result"` key
is used to indicate success from Parity.
@ -351,6 +370,7 @@ defmodule EthereumJSONRPC.Parity.Trace do
...> "subtraces" => 0,
...> "traceAddress" => [0],
...> "transactionHash" => "0xb012b8c53498c669d87d85ed90f57385848b86d3f44ed14b2784ec685d6fda98",
...> "transactionIndex" => 0,
...> "type" => "suicide"
...> }
...> )
@ -366,6 +386,7 @@ defmodule EthereumJSONRPC.Parity.Trace do
"subtraces" => 0,
"traceAddress" => [0],
"transactionHash" => "0xb012b8c53498c669d87d85ed90f57385848b86d3f44ed14b2784ec685d6fda98",
"transactionIndex" => 0,
"type" => "suicide"
}
@ -387,6 +408,7 @@ defmodule EthereumJSONRPC.Parity.Trace do
...> "subtraces" => 7,
...> "traceAddress" => [],
...> "transactionHash" => "0xcd7c15dbbc797722bef6e1d551edfd644fc7f4fb2ccd6a7947b2d1ade9ed140b",
...> "transactionIndex" => 0,
...> "type" => "call"
...> }
...> )
@ -405,19 +427,20 @@ defmodule EthereumJSONRPC.Parity.Trace do
"subtraces" => 7,
"traceAddress" => [],
"transactionHash" => "0xcd7c15dbbc797722bef6e1d551edfd644fc7f4fb2ccd6a7947b2d1ade9ed140b",
"transactionIndex" => 0,
"type" => "call"
}
"""
def to_elixir(%{"blockNumber" => _, "index" => _, "transactionHash" => _} = trace)
def to_elixir(%{"blockNumber" => _, "index" => _, "transactionHash" => _, "transactionIndex" => _} = trace)
when is_map(trace) do
Enum.into(trace, %{}, &entry_to_elixir/1)
end
def to_elixir(_) do
raise ArgumentError,
~S|Caller must `Map.put/2` `"blockNumber"`, `"index"`, and `"transactionHash"` in trace|
~S|Caller must `Map.put/2` `"blockNumber"`, `"index"`, `"transactionHash"` and `"transactionIndex"` in trace|
end
# subtraces is an actual integer in JSON and not hex-encoded
@ -441,6 +464,8 @@ defmodule EthereumJSONRPC.Parity.Trace do
{key, Result.to_elixir(result)}
end
defp entry_to_elixir({"transactionIndex", index} = entry) when is_integer(index), do: entry
defp put_call_error_or_result(params, %{
"result" => %{"gasUsed" => gas_used, "output" => output}
}) do

@ -60,7 +60,8 @@ defmodule EthereumJSONRPC.Transaction do
s: non_neg_integer(),
to_address_hash: EthereumJSONRPC.address(),
v: non_neg_integer(),
value: non_neg_integer()
value: non_neg_integer(),
transaction_index: non_neg_integer()
}
@doc """
@ -98,7 +99,8 @@ defmodule EthereumJSONRPC.Transaction do
s: 31606574786494953692291101914709926755545765281581808821704454381804773090106,
to_address_hash: "0x5df9b87991262f6ba471f09758cde1c0fc1de734",
v: 28,
value: 31337
value: 31337,
transaction_index: 0
}
"""
@ -133,7 +135,8 @@ defmodule EthereumJSONRPC.Transaction do
s: s,
to_address_hash: to_address_hash,
v: v,
value: value
value: value,
transaction_index: index
}
end

@ -55,7 +55,8 @@ defmodule EthereumJSONRPC.Transactions do
s: "0x72caddc0371451a58de2ca6ab64e0f586ccdb9465ff54e1c82564940e89291e3",
to_address_hash: nil,
v: "0xbd",
value: 0
value: 0,
transaction_index: 0
}
]

@ -179,7 +179,9 @@ defmodule EthereumJSONRPC.HTTP.MoxTest do
end
transactions_params =
Enum.map(transaction_hashes, fn hash_data -> %{block_number: 0, hash_data: hash_data, gas: 1_000_000} end)
Enum.map(transaction_hashes, fn hash_data ->
%{block_number: 0, hash_data: hash_data, gas: 1_000_000, transaction_index: 0}
end)
assert {:ok, responses} =
EthereumJSONRPC.fetch_internal_transactions(transactions_params, json_rpc_named_arguments)

@ -32,6 +32,7 @@ defmodule EthereumJSONRPC.ParityTest do
gas_used = 382_953
trace_address = []
transaction_hash = "0x0fa6f723216dba694337f9bb37d8870725655bdf2573526a39454685659e39b1"
transaction_index = 0
type = "create"
if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do
@ -58,6 +59,7 @@ defmodule EthereumJSONRPC.ParityTest do
},
"traceAddress" => trace_address,
"transactionHash" => transaction_hash,
"transactionIndex" => transaction_index,
"type" => type
}
]
@ -71,7 +73,8 @@ defmodule EthereumJSONRPC.ParityTest do
[
%{
block_number: block_number,
hash_data: transaction_hash
hash_data: transaction_hash,
transaction_index: transaction_index
}
],
json_rpc_named_arguments
@ -90,7 +93,8 @@ defmodule EthereumJSONRPC.ParityTest do
trace_address: trace_address,
transaction_hash: transaction_hash,
type: type,
value: value
value: value,
transaction_index: transaction_index
}
]
}
@ -118,7 +122,8 @@ defmodule EthereumJSONRPC.ParityTest do
[
%{
block_number: 1,
hash_data: "0x0000000000000000000000000000000000000000000000000000000000000001"
hash_data: "0x0000000000000000000000000000000000000000000000000000000000000001",
transaction_index: 0
}
],
json_rpc_named_arguments
@ -129,7 +134,8 @@ defmodule EthereumJSONRPC.ParityTest do
code: -32603,
data: %{
"blockNumber" => 1,
"transactionHash" => "0x0000000000000000000000000000000000000000000000000000000000000001"
"transactionHash" => "0x0000000000000000000000000000000000000000000000000000000000000001",
"transactionIndex" => 0
},
message:
"Internal error occurred: {}, this should not be the case with eth_call, most likely a bug."
@ -179,22 +185,26 @@ defmodule EthereumJSONRPC.ParityTest do
# start with :ok
%{
block_number: 1,
hash_data: "0x0fa6f723216dba694337f9bb37d8870725655bdf2573526a39454685659e39b1"
hash_data: "0x0fa6f723216dba694337f9bb37d8870725655bdf2573526a39454685659e39b1",
transaction_index: 0
},
# :ok, :ok clause
%{
block_number: 34,
hash_data: "0x3a3eb134e6792ce9403ea4188e5e79693de9e4c94e499db132be086400da79e6"
hash_data: "0x3a3eb134e6792ce9403ea4188e5e79693de9e4c94e499db132be086400da79e6",
transaction_index: 0
},
# :ok, :error clause
%{
block_number: 1,
hash_data: "0x0000000000000000000000000000000000000000000000000000000000000001"
hash_data: "0x0000000000000000000000000000000000000000000000000000000000000001",
transaction_index: 0
},
# :error, :error clause
%{
block_number: 2,
hash_data: "0x0000000000000000000000000000000000000000000000000000000000000002"
hash_data: "0x0000000000000000000000000000000000000000000000000000000000000002",
transaction_index: 0
}
],
json_rpc_named_arguments
@ -205,7 +215,8 @@ defmodule EthereumJSONRPC.ParityTest do
code: -32603,
data: %{
"blockNumber" => 1,
"transactionHash" => "0x0000000000000000000000000000000000000000000000000000000000000001"
"transactionHash" => "0x0000000000000000000000000000000000000000000000000000000000000001",
"transactionIndex" => 0
},
message:
"Internal error occurred: {}, this should not be the case with eth_call, most likely a bug."
@ -214,7 +225,8 @@ defmodule EthereumJSONRPC.ParityTest do
code: -32603,
data: %{
"blockNumber" => 2,
"transactionHash" => "0x0000000000000000000000000000000000000000000000000000000000000002"
"transactionHash" => "0x0000000000000000000000000000000000000000000000000000000000000002",
"transactionIndex" => 0
},
message:
"Internal error occurred: {}, this should not be the case with eth_call, most likely a bug."

@ -150,7 +150,8 @@ defmodule Explorer.Chain.InternalTransaction do
...> trace_address: [],
...> transaction_hash: "0x3c624bb4852fb5e35a8f45644cec7a486211f6ba89034768a2b763194f22f97d",
...> type: "create",
...> value: 0
...> value: 0,
...> transaction_index: 0
...> }
iex> )
iex> changeset.valid?

@ -579,7 +579,8 @@ defmodule Explorer.Chain.ImportTest do
trace_address: [],
transaction_hash: transaction_hash,
type: "call",
value: 0
value: 0,
transaction_index: 0
}
]
}
@ -672,7 +673,8 @@ defmodule Explorer.Chain.ImportTest do
trace_address: [],
transaction_hash: transaction_hash,
type: "create",
value: 0
value: 0,
transaction_index: 0
}
]
}
@ -948,7 +950,8 @@ defmodule Explorer.Chain.ImportTest do
trace_address: [],
transaction_hash: "0x1a263224a95275d77bc30a7e131bc64d948777946a790c0915ab293791fbcb61",
type: "create",
value: 0
value: 0,
transaction_index: 0
},
%{
block_number: 6_546_180,
@ -960,7 +963,8 @@ defmodule Explorer.Chain.ImportTest do
trace_address: [],
transaction_hash: "0xab349efbe1ddc6d85d84a993aa52bdaadce66e8ee166dd10013ce3f2a94ca724",
type: "create",
value: 0
value: 0,
transaction_index: 0
}
]
}

@ -127,8 +127,9 @@ defmodule Indexer.Block.Catchup.Fetcher do
}) do
transactions
|> Enum.map(fn transaction_hash ->
block_number = Map.fetch!(transaction_hash_to_block_number, to_string(transaction_hash))
%{block_number: block_number, hash: transaction_hash}
transaction = Map.fetch!(transaction_hash_to_block_number, to_string(transaction_hash))
%{block_number: transaction[:block_number], hash: transaction_hash, index: transaction[:index]}
end)
|> InternalTransaction.Fetcher.async_fetch(10_000)
end

@ -218,8 +218,8 @@ defmodule Indexer.Block.Fetcher do
defp get_transaction_hash_to_block_number(options) do
options
|> get_in([:transactions, :params, Access.all()])
|> Enum.into(%{}, fn %{block_number: block_number, hash: hash} ->
{hash, block_number}
|> Enum.into(%{}, fn %{block_number: block_number, hash: hash, index: index} ->
{hash, %{block_number: block_number, index: index}}
end)
end

@ -234,9 +234,13 @@ defmodule Indexer.Block.Realtime.Fetcher do
Enum.map(transactions_params, &transaction_params_to_fetch_internal_transaction_params/1)
end
defp transaction_params_to_fetch_internal_transaction_params(%{block_number: block_number, hash: hash})
defp transaction_params_to_fetch_internal_transaction_params(%{
block_number: block_number,
hash: hash,
transaction_index: transaction_index
})
when is_integer(block_number) do
%{block_number: block_number, hash_data: to_string(hash)}
%{block_number: block_number, hash_data: to_string(hash), transaction_index: transaction_index}
end
defp balances(

@ -70,7 +70,7 @@ defmodule Indexer.InternalTransaction.Fetcher do
def init(initial, reducer, _) do
{:ok, final} =
Chain.stream_transactions_with_unfetched_internal_transactions(
[:block_number, :hash],
[:block_number, :hash, :index],
initial,
fn transaction_fields, acc ->
transaction_fields
@ -82,13 +82,13 @@ defmodule Indexer.InternalTransaction.Fetcher do
final
end
defp entry(%{block_number: block_number, hash: %Hash{bytes: bytes}}) when is_integer(block_number) do
{block_number, bytes}
defp entry(%{block_number: block_number, hash: %Hash{bytes: bytes}, index: index}) when is_integer(block_number) do
{block_number, bytes, index}
end
defp params({block_number, hash_bytes}) when is_integer(block_number) do
defp params({block_number, hash_bytes, index}) when is_integer(block_number) do
{:ok, hash} = Hash.Full.cast(hash_bytes)
%{block_number: block_number, hash_data: to_string(hash)}
%{block_number: block_number, hash_data: to_string(hash), transaction_index: index}
end
@impl BufferedTask

@ -101,7 +101,7 @@ defmodule Indexer.InternalTransaction.FetcherTest do
[],
fn hash_string, acc -> [hash_string | acc] end,
json_rpc_named_arguments
) == [{block.number, collated_unfetched_transaction.hash.bytes}]
) == [{block.number, collated_unfetched_transaction.hash.bytes, collated_unfetched_transaction.index}]
end
test "does not buffer collated transactions with fetched internal transactions", %{
@ -136,8 +136,8 @@ defmodule Indexer.InternalTransaction.FetcherTest do
capture_log(fn ->
InternalTransaction.Fetcher.run(
[
{1, bytes},
{1, bytes}
{1, bytes, 0},
{1, bytes, 0}
],
0,
json_rpc_named_arguments
@ -147,8 +147,8 @@ defmodule Indexer.InternalTransaction.FetcherTest do
assert log =~
"""
Duplicate entries being used to fetch internal transactions:
1. {1, <<3, 205, 88, 153, 166, 59, 111, 98, 34, 175, 218, 135, 5, 208, 89, 253, 90, 125, 18, 107, 202, 190, 150, 47, 182, 84, 217, 115, 110, 107, 202, 250>>}
2. {1, <<3, 205, 88, 153, 166, 59, 111, 98, 34, 175, 218, 135, 5, 208, 89, 253, 90, 125, 18, 107, 202, 190, 150, 47, 182, 84, 217, 115, 110, 107, 202, 250>>}
1. {1, <<3, 205, 88, 153, 166, 59, 111, 98, 34, 175, 218, 135, 5, 208, 89, 253, 90, 125, 18, 107, 202, 190, 150, 47, 182, 84, 217, 115, 110, 107, 202, 250>>, 0}
2. {1, <<3, 205, 88, 153, 166, 59, 111, 98, 34, 175, 218, 135, 5, 208, 89, 253, 90, 125, 18, 107, 202, 190, 150, 47, 182, 84, 217, 115, 110, 107, 202, 250>>, 0}
"""
end
@ -167,12 +167,12 @@ defmodule Indexer.InternalTransaction.FetcherTest do
assert InternalTransaction.Fetcher.run(
[
{1, bytes},
{1, bytes}
{1, bytes, 0},
{1, bytes, 0}
],
0,
json_rpc_named_arguments
) == {:retry, [{1, bytes}]}
) == {:retry, [{1, bytes, 0}]}
end
end
end

Loading…
Cancel
Save