Update block reward contract beneficiary balances

Why:

* "Parity’s consensus engine allows using a smart contract for block
reward calculation." - see https://wiki.parity.io/Block-Reward-Contract
for details. This means that we need to use `trace_block`
(https://wiki.parity.io/JSONRPC-trace-module#trace_block) to fetch the
block reward contract beneficiaries and update their balances. Currently
this doesn't happen so there are addresses (block contract reward
beneficiaries) with inaccurate balances and missing rows in the
`address_coin_balances` table.
* Issue link: https://github.com/poanetwork/blockscout/issues/767

This change addresses the need by:

* Adding `EthereumJSONRPC.fetch_beneficiaries/2` to help us fetch the
block reward contract beneficiaries of a given block range. This only
works with Parity.
* Editing Indexer's dev and prod config by adding `trace_block` config to
`:method_to_url`.
* Adding `block_reward_contract_beneficiaries` to
`Indexer.AddressExtraction` for it to know how to get addresses in this
scenario.
* Editing `Indexer.Block.Fetcher.fetch_and_import_range/2` to get block
reward contract beneficiaries, create their addresses if they don't yet
exist, and update their coin balances.
pull/936/head
Sebastian Abondano 6 years ago
parent 4199999e63
commit 2b8ceb4e32
  1. 7
      apps/ethereum_jsonrpc/lib/ethereum_jsonrpc.ex
  2. 8
      apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/geth.ex
  3. 45
      apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/parity.ex
  4. 16
      apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/variant.ex
  5. 274
      apps/ethereum_jsonrpc/test/ethereum_jsonrpc/parity_test.exs
  6. 13
      apps/ethereum_jsonrpc/test/ethereum_jsonrpc_test.exs
  7. 1
      apps/indexer/config/dev/parity.exs
  8. 1
      apps/indexer/config/prod/parity.exs
  9. 12
      apps/indexer/lib/indexer/address_extraction.ex
  10. 18
      apps/indexer/lib/indexer/block/fetcher.ex
  11. 12
      apps/indexer/test/indexer/address_extraction_test.exs
  12. 14
      apps/indexer/test/indexer/block/catchup/bound_interval_supervisor_test.exs
  13. 7
      apps/indexer/test/indexer/block/fetcher_test.exs
  14. 6
      apps/indexer/test/indexer/block/realtime/fetcher_test.exs

@ -187,6 +187,13 @@ defmodule EthereumJSONRPC do
end
end
@doc """
Fetches block reward contract beneficiaries from variant API.
"""
def fetch_beneficiaries(_first.._last = range, json_rpc_named_arguments) do
Keyword.fetch!(json_rpc_named_arguments, :variant).fetch_beneficiaries(range, json_rpc_named_arguments)
end
@doc """
Fetches blocks by block hashes.

@ -5,6 +5,14 @@ defmodule EthereumJSONRPC.Geth do
@behaviour EthereumJSONRPC.Variant
@doc """
Block reward contract beneficiary fetching is not supported currently for Geth.
To signal to the caller that fetching is not supported, `:ignore` is returned.
"""
@impl EthereumJSONRPC.Variant
def fetch_beneficiaries(_block_range, _json_rpc_named_arguments), do: :ignore
@doc """
Internal transaction fetching is not supported currently for Geth.

@ -10,6 +10,30 @@ defmodule EthereumJSONRPC.Parity do
@behaviour EthereumJSONRPC.Variant
@impl EthereumJSONRPC.Variant
def fetch_beneficiaries(block_range, json_rpc_named_arguments) do
Enum.reduce(
Enum.with_index(block_range),
{:ok, MapSet.new()},
fn
{block_number, index}, {:ok, beneficiaries} ->
quantity = EthereumJSONRPC.integer_to_quantity(block_number)
case trace_block(index, quantity, json_rpc_named_arguments) do
{:ok, traces} when is_list(traces) ->
new_beneficiaries = extract_beneficiaries(traces)
{:ok, MapSet.union(new_beneficiaries, beneficiaries)}
_ ->
{:error, "Error fetching block reward contract beneficiaries"}
end
_, {:error, _} = error ->
error
end
)
end
@doc """
Fetches the `t:Explorer.Chain.InternalTransaction.changeset/2` params from the Parity trace URL.
"""
@ -48,6 +72,27 @@ defmodule EthereumJSONRPC.Parity do
end
end
defp extract_beneficiaries(traces) when is_list(traces) do
Enum.reduce(traces, MapSet.new(), fn
%{"action" => %{"rewardType" => "block", "author" => author}, "blockNumber" => block_number}, beneficiaries ->
beneficiary = %{
block_number: block_number,
address_hash: author
}
MapSet.put(beneficiaries, beneficiary)
_, beneficiaries ->
beneficiaries
end)
end
defp trace_block(index, quantity, json_rpc_named_arguments) do
%{id: index, method: "trace_block", params: [quantity]}
|> request()
|> json_rpc(json_rpc_named_arguments)
end
defp trace_replay_transaction_responses_to_internal_transactions_params(responses, id_to_params)
when is_list(responses) and is_map(id_to_params) do
with {:ok, traces} <- trace_replay_transaction_responses_to_traces(responses, id_to_params) do

@ -13,6 +13,22 @@ defmodule EthereumJSONRPC.Variant do
@type internal_transaction_params :: map()
@doc """
Fetch the block reward contract beneficiaries for a given block
range, from the variant of the Ethereum JSONRPC API.
For more information on block reward contracts see:
https://wiki.parity.io/Block-Reward-Contract.html
## Returns
* `{:ok, #MapSet<[%{...}]>}` - beneficiaries were successfully fetched
* `{:error, reason}` - there was one or more errors with `reason` in fetching the beneficiaries
* `:ignore` - the variant does not support fetching beneficiaries
"""
@callback fetch_beneficiaries(Range.t(), EthereumJSONRPC.json_rpc_named_arguments()) ::
{:ok, MapSet.t()} | {:error, reason :: term} | :ignore
@doc """
Fetches the `t:Explorer.Chain.InternalTransaction.changeset/2` params from the variant of the Ethereum JSONRPC API.

@ -222,4 +222,278 @@ defmodule EthereumJSONRPC.ParityTest do
]}
end
end
describe "fetch_beneficiaries/1" do
test "with valid block range, returns {:ok, addresses}", %{
json_rpc_named_arguments: json_rpc_named_arguments
} do
block_number = 5_080_887
block_quantity = EthereumJSONRPC.integer_to_quantity(block_number)
hash1 = "0xef481b4e2c3ed62265617f2e9dfcdf3cf3efc11a"
hash2 = "0x523b6539ff08d72a6c8bb598af95bf50c1ea839c"
if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do
expect(EthereumJSONRPC.Mox, :json_rpc, fn %{params: [^block_quantity]}, _options ->
{:ok,
[
%{
"action" => %{
"author" => hash1,
"rewardType" => "block",
"value" => "0xde0b6b3a7640000"
},
"blockHash" => "0x52a8d2185282506ce681364d2aa0c085ba45fdeb5d6c0ddec1131617a71ee2ca",
"blockNumber" => block_number,
"result" => nil,
"subtraces" => 0,
"traceAddress" => [],
"transactionHash" => nil,
"transactionPosition" => nil,
"type" => "reward"
},
%{
"action" => %{
"author" => hash2,
"rewardType" => "block",
"value" => "0xde0b6b3a7640000"
},
"blockHash" => "0x52a8d2185282506ce681364d2aa0c085ba45fdeb5d6c0ddec1131617a71ee2ca",
"blockNumber" => block_number,
"result" => nil,
"subtraces" => 0,
"traceAddress" => [],
"transactionHash" => nil,
"transactionPosition" => nil,
"type" => "reward"
}
]}
end)
end
expected_beneficiaries =
MapSet.new([
%{block_number: block_number, address_hash: hash2},
%{block_number: block_number, address_hash: hash1}
])
{:ok, fetched_beneficiaries} =
EthereumJSONRPC.Parity.fetch_beneficiaries(5_080_887..5_080_887, json_rpc_named_arguments)
assert fetched_beneficiaries == expected_beneficiaries
end
test "with no rewards, returns {:ok, []}", %{
json_rpc_named_arguments: json_rpc_named_arguments
} do
if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do
expect(EthereumJSONRPC.Mox, :json_rpc, fn _json, _options ->
{:ok, []}
end)
{:ok, fetched_beneficiaries} =
EthereumJSONRPC.Parity.fetch_beneficiaries(5_080_887..5_080_887, json_rpc_named_arguments)
assert fetched_beneficiaries == MapSet.new()
end
end
test "with nil rewards, returns {:error, }", %{
json_rpc_named_arguments: json_rpc_named_arguments
} do
if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do
expect(EthereumJSONRPC.Mox, :json_rpc, fn _json, _options ->
{:ok, nil}
end)
result = EthereumJSONRPC.Parity.fetch_beneficiaries(5_080_887..5_080_887, json_rpc_named_arguments)
assert result == {:error, "Error fetching block reward contract beneficiaries"}
end
end
test "ignores non-reward traces", %{
json_rpc_named_arguments: json_rpc_named_arguments
} do
block_number = 5_077_429
block_quantity = EthereumJSONRPC.integer_to_quantity(block_number)
hash1 = "0xcfa53498686e00d3b4b41f3bea61604038eebb58"
hash2 = "0x523b6539ff08d72a6c8bb598af95bf50c1ea839c"
if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do
expect(EthereumJSONRPC.Mox, :json_rpc, fn %{params: [^block_quantity]}, _options ->
{:ok,
[
%{
"action" => %{
"callType" => "call",
"from" => "0x95426f2bc716022fcf1def006dbc4bb81f5b5164",
"gas" => "0x0",
"input" => "0x",
"to" => "0xe797a1da01eb0f951e0e400f9343de9d17a06bac",
"value" => "0x4a817c800"
},
"blockHash" => "0x6659a4926d833a7eab74379fa647ec74c9f5e65f8029552a35264126560f300a",
"blockNumber" => block_number,
"result" => %{"gasUsed" => "0x0", "output" => "0x"},
"subtraces" => 0,
"traceAddress" => [],
"transactionHash" => "0x5acf90f846b8216bdbc309cf4eb24adc69d730bf29304dc0e740cf6df850666e",
"transactionPosition" => 0,
"type" => "call"
},
%{
"action" => %{
"author" => hash1,
"rewardType" => "block",
"value" => "0xde0b6b3a7640000"
},
"blockHash" => "0x6659a4926d833a7eab74379fa647ec74c9f5e65f8029552a35264126560f300a",
"blockNumber" => block_number,
"result" => nil,
"subtraces" => 0,
"traceAddress" => [],
"transactionHash" => nil,
"transactionPosition" => nil,
"type" => "reward"
},
%{
"action" => %{
"author" => hash2,
"rewardType" => "block",
"value" => "0xde0b6b3a7640000"
},
"blockHash" => "0x6659a4926d833a7eab74379fa647ec74c9f5e65f8029552a35264126560f300a",
"blockNumber" => block_number,
"result" => nil,
"subtraces" => 0,
"traceAddress" => [],
"transactionHash" => nil,
"transactionPosition" => nil,
"type" => "reward"
}
]}
end)
end
expected_beneficiaries =
MapSet.new([
%{block_number: block_number, address_hash: hash2},
%{block_number: block_number, address_hash: hash1}
])
{:ok, fetched_beneficiaries} =
EthereumJSONRPC.Parity.fetch_beneficiaries(5_077_429..5_077_429, json_rpc_named_arguments)
assert fetched_beneficiaries == expected_beneficiaries
end
test "with multiple blocks with repeat beneficiaries", %{
json_rpc_named_arguments: json_rpc_named_arguments
} do
block_number1 = 5_080_886
block_quantity1 = EthereumJSONRPC.integer_to_quantity(block_number1)
block_number2 = 5_080_887
block_quantity2 = EthereumJSONRPC.integer_to_quantity(block_number2)
hash1 = "0xadc702c4bb09fbc502dd951856b9c7a1528a88de"
hash2 = "0xef481b4e2c3ed62265617f2e9dfcdf3cf3efc11a"
hash3 = "0x523b6539ff08d72a6c8bb598af95bf50c1ea839c"
if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do
expect(EthereumJSONRPC.Mox, :json_rpc, 2, fn
%{params: [^block_quantity1]} = _json, _options ->
{:ok,
[
%{
"action" => %{
"author" => hash1,
"rewardType" => "block",
"value" => "0xde0b6b3a7640000"
},
"blockNumber" => block_number1,
"result" => nil,
"subtraces" => 0,
"traceAddress" => [],
"transactionHash" => nil,
"transactionPosition" => nil,
"type" => "reward"
},
%{
"action" => %{
"author" => hash3,
"rewardType" => "block",
"value" => "0xde0b6b3a7640000"
},
"blockNumber" => block_number1,
"result" => nil,
"subtraces" => 0,
"traceAddress" => [],
"transactionHash" => nil,
"transactionPosition" => nil,
"type" => "reward"
}
]}
%{params: [^block_quantity2]} = _json, _options ->
{:ok,
[
%{
"action" => %{
"author" => hash2,
"rewardType" => "block",
"value" => "0xde0b6b3a7640000"
},
"blockNumber" => block_number2,
"result" => nil,
"subtraces" => 0,
"traceAddress" => [],
"transactionHash" => nil,
"transactionPosition" => nil,
"type" => "reward"
},
%{
"action" => %{
"author" => hash3,
"rewardType" => "block",
"value" => "0xde0b6b3a7640000"
},
"blockNumber" => block_number2,
"result" => nil,
"subtraces" => 0,
"traceAddress" => [],
"transactionHash" => nil,
"transactionPosition" => nil,
"type" => "reward"
}
]}
end)
end
expected_beneficiaries =
MapSet.new([
%{block_number: block_number1, address_hash: hash3},
%{block_number: block_number2, address_hash: hash3},
%{block_number: block_number2, address_hash: hash2},
%{block_number: block_number1, address_hash: hash1}
])
{:ok, fetched_beneficiaries} =
EthereumJSONRPC.Parity.fetch_beneficiaries(5_080_886..5_080_887, json_rpc_named_arguments)
assert fetched_beneficiaries == expected_beneficiaries
end
test "with error, returns {:error, reason}", %{
json_rpc_named_arguments: json_rpc_named_arguments
} do
if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do
expect(EthereumJSONRPC.Mox, :json_rpc, fn _json, _options ->
{:error, "oops"}
end)
result = EthereumJSONRPC.Parity.fetch_beneficiaries(5_080_887..5_080_887, json_rpc_named_arguments)
assert result == {:error, "Error fetching block reward contract beneficiaries"}
end
end
end
end

@ -165,6 +165,19 @@ defmodule EthereumJSONRPCTest do
end
end
describe "fetch_beneficiaries/2" do
@tag :no_geth
test "fetches benefeciaries from variant API", %{json_rpc_named_arguments: json_rpc_named_arguments} do
if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do
expect(EthereumJSONRPC.Mox, :json_rpc, fn _, _ ->
{:ok, []}
end)
assert EthereumJSONRPC.fetch_beneficiaries(1..1, json_rpc_named_arguments) == {:ok, MapSet.new()}
end
end
end
describe "fetch_block_by_hash/2" do
test "can fetch blocks", %{json_rpc_named_arguments: json_rpc_named_arguments} do
%{block_hash: block_hash, transaction_hash: transaction_hash} =

@ -9,6 +9,7 @@ config :indexer,
url: "https://sokol.poa.network",
method_to_url: [
eth_getBalance: "https://sokol-trace.poa.network",
trace_block: "https://sokol-trace.poa.network",
trace_replayTransaction: "https://sokol-trace.poa.network"
],
http_options: [recv_timeout: 60_000, timeout: 60_000, hackney: [pool: :ethereum_jsonrpc]]

@ -9,6 +9,7 @@ config :indexer,
url: System.get_env("ETHEREUM_URL") || "https://sokol.poa.network",
method_to_url: [
eth_getBalance: System.get_env("TRACE_URL") || "https://sokol-trace.poa.network",
trace_block: System.get_env("TRACE_URL") || "https://sokol-trace.poa.network",
trace_replayTransaction: System.get_env("TRACE_URL") || "https://sokol-trace.poa.network"
],
http_options: [recv_timeout: 60_000, timeout: 60_000, hackney: [pool: :ethereum_jsonrpc]]

@ -116,6 +116,12 @@ defmodule Indexer.AddressExtraction do
%{from: :block_number, to: :fetched_coin_balance_block_number},
%{from: :to_address_hash, to: :hash}
]
],
block_reward_contract_beneficiaries: [
[
%{from: :block_number, to: :fetched_coin_balance_block_number},
%{from: :address_hash, to: :hash}
]
]
}
@ -379,6 +385,12 @@ defmodule Indexer.AddressExtraction do
required(:to_address_hash) => String.t(),
required(:block_number) => non_neg_integer()
}
],
optional(:block_reward_contract_beneficiaries) => [
%{
required(:address_hash) => String.t(),
required(:block_number) => non_neg_integer()
}
]
}) :: [params]
def extract_addresses(fetched_data, options \\ []) when is_map(fetched_data) and is_list(options) do

@ -101,8 +101,10 @@ defmodule Indexer.Block.Fetcher do
transactions_with_receipts = Receipts.put(transactions_without_receipts, receipts),
%{token_transfers: token_transfers, tokens: tokens} = TokenTransfers.parse(logs),
%{mint_transfers: mint_transfers} = MintTransfer.parse(logs),
{:beneficiaries, {:ok, beneficiaries}} <- fetch_beneficiaries(range, json_rpc_named_arguments),
addresses =
AddressExtraction.extract_addresses(%{
block_reward_contract_beneficiaries: MapSet.to_list(beneficiaries),
blocks: blocks,
logs: logs,
mint_transfers: mint_transfers,
@ -110,11 +112,13 @@ defmodule Indexer.Block.Fetcher do
transactions: transactions_with_receipts
}),
coin_balances_params_set =
CoinBalances.params_set(%{
%{
blocks_params: blocks,
logs_params: logs,
transactions_params: transactions_with_receipts
}),
}
|> CoinBalances.params_set()
|> MapSet.union(beneficiaries),
address_token_balances = TokenBalances.params_set(%{token_transfers_params: token_transfers}),
{:ok, inserted} <-
__MODULE__.import(
@ -191,6 +195,16 @@ defmodule Indexer.Block.Fetcher do
def async_import_uncles(_), do: :ok
defp fetch_beneficiaries(range, json_rpc_named_arguments) do
result =
case EthereumJSONRPC.fetch_beneficiaries(range, json_rpc_named_arguments) do
:ignore -> {:ok, MapSet.new()}
result -> result
end
{:beneficiaries, result}
end
# `fetched_balance_block_number` is needed for the `CoinBalanceFetcher`, but should not be used for `import` because the
# balance is not known yet.
defp pop_address_hash_to_fetched_balance_block_number(options) do

@ -107,12 +107,18 @@ defmodule Indexer.AddressExtractionTest do
token_contract_address_hash: gen_hash()
}
beneficiary = %{
block_number: 6,
address_hash: gen_hash()
}
blockchain_data = %{
blocks: [block],
internal_transactions: [internal_transaction],
transactions: [transaction],
logs: [log],
token_transfers: [token_transfer]
token_transfers: [token_transfer],
block_reward_contract_beneficiaries: [beneficiary]
}
assert AddressExtraction.extract_addresses(blockchain_data) == [
@ -144,6 +150,10 @@ defmodule Indexer.AddressExtractionTest do
%{
hash: token_transfer.token_contract_address_hash,
fetched_coin_balance_block_number: token_transfer.block_number
},
%{
hash: beneficiary.address_hash,
fetched_coin_balance_block_number: beneficiary.block_number
}
]
end

@ -63,6 +63,9 @@ defmodule Indexer.Block.Catchup.BoundIntervalSupervisorTest do
"uncles" => []
}}
%{method: "trace_block"}, _options ->
{:ok, []}
[%{method: "eth_getBlockByNumber", params: [_, true]} | _] = requests, _options ->
{:ok,
Enum.map(requests, fn %{id: id, params: [block_quantity, true]} ->
@ -464,6 +467,17 @@ defmodule Indexer.Block.Catchup.BoundIntervalSupervisorTest do
}
]}
end)
|> (fn mock ->
case Keyword.fetch!(json_rpc_named_arguments, :variant) do
EthereumJSONRPC.Parity ->
expect(mock, :json_rpc, fn %{method: "trace_block"}, _options ->
{:ok, []}
end)
_ ->
mock
end
end).()
|> stub(:json_rpc, fn [
%{
id: id,

@ -109,6 +109,9 @@ defmodule Indexer.Block.FetcherTest do
}
]}
end)
|> expect(:json_rpc, fn %{id: _id, method: "trace_block", params: [^block_quantity]}, _options ->
{:ok, []}
end)
|> expect(:json_rpc, fn [
%{
id: id,
@ -356,6 +359,10 @@ defmodule Indexer.Block.FetcherTest do
}
]}
end)
|> expect(:json_rpc, fn json, _options ->
assert %{id: _id, method: "trace_block", params: [^block_quantity]} = json
{:ok, []}
end)
# async requests need to be grouped in one expect because the order is non-deterministic while multiple expect
# calls on the same name/arity are used in order
|> expect(:json_rpc, 5, fn json, _options ->

@ -24,7 +24,8 @@ defmodule Indexer.Block.Realtime.FetcherTest do
|> put_in(
[:transport_options, :method_to_url],
eth_getBalance: "https://core-trace.poa.network",
trace_replayTransaction: "https://core-trace.poa.network"
trace_replayTransaction: "https://core-trace.poa.network",
trace_block: "https://core-trace.poa.network"
)
block_fetcher = %Indexer.Block.Fetcher{
@ -195,6 +196,9 @@ defmodule Indexer.Block.Realtime.FetcherTest do
}
]}
end)
|> expect(:json_rpc, 2, fn %{method: "trace_block"}, _options ->
{:ok, []}
end)
|> expect(:json_rpc, fn [
%{
id: 0,

Loading…
Cancel
Save