Merge pull request #936 from poanetwork/sa-block-reward-contract-support

Update block reward contract beneficiary balances

apps/ethereum_jsonrpc/lib/ethereum_jsonrpc.ex

@@ -187,6 +187,13 @@ defmodule EthereumJSONRPC do
     end
   end
 
+  @doc """
+  Fetches block reward contract beneficiaries from variant API.
+  """
+  def fetch_beneficiaries(_first.._last = range, json_rpc_named_arguments) do
+    Keyword.fetch!(json_rpc_named_arguments, :variant).fetch_beneficiaries(range, json_rpc_named_arguments)
+  end
+
   @doc """
   Fetches blocks by block hashes.
 

apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/geth.ex

@@ -5,6 +5,14 @@ defmodule EthereumJSONRPC.Geth do
 
   @behaviour EthereumJSONRPC.Variant
 
+  @doc """
+  Block reward contract beneficiary fetching is not supported currently for Geth.
+
+  To signal to the caller that fetching is not supported, `:ignore` is returned.
+  """
+  @impl EthereumJSONRPC.Variant
+  def fetch_beneficiaries(_block_range, _json_rpc_named_arguments), do: :ignore
+
   @doc """
   Internal transaction fetching is not supported currently for Geth.
 

apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/parity.ex

@@ -10,6 +10,30 @@ defmodule EthereumJSONRPC.Parity do
 
   @behaviour EthereumJSONRPC.Variant
 
+  @impl EthereumJSONRPC.Variant
+  def fetch_beneficiaries(block_range, json_rpc_named_arguments) do
+    Enum.reduce(
+      Enum.with_index(block_range),
+      {:ok, MapSet.new()},
+      fn
+        {block_number, index}, {:ok, beneficiaries} ->
+          quantity = EthereumJSONRPC.integer_to_quantity(block_number)
+
+          case trace_block(index, quantity, json_rpc_named_arguments) do
+            {:ok, traces} when is_list(traces) ->
+              new_beneficiaries = extract_beneficiaries(traces)
+              {:ok, MapSet.union(new_beneficiaries, beneficiaries)}
+
+            _ ->
+              {:error, "Error fetching block reward contract beneficiaries"}
+          end
+
+        _, {:error, _} = error ->
+          error
+      end
+    )
+  end
+
   @doc """
   Fetches the `t:Explorer.Chain.InternalTransaction.changeset/2` params from the Parity trace URL.
   """
@@ -48,6 +72,27 @@ defmodule EthereumJSONRPC.Parity do
     end
   end
 
+  defp extract_beneficiaries(traces) when is_list(traces) do
+    Enum.reduce(traces, MapSet.new(), fn
+      %{"action" => %{"rewardType" => "block", "author" => author}, "blockNumber" => block_number}, beneficiaries ->
+        beneficiary = %{
+          block_number: block_number,
+          address_hash: author
+        }
+
+        MapSet.put(beneficiaries, beneficiary)
+
+      _, beneficiaries ->
+        beneficiaries
+    end)
+  end
+
+  defp trace_block(index, quantity, json_rpc_named_arguments) do
+    %{id: index, method: "trace_block", params: [quantity]}
+    |> request()
+    |> json_rpc(json_rpc_named_arguments)
+  end
+
   defp trace_replay_transaction_responses_to_internal_transactions_params(responses, id_to_params)
        when is_list(responses) and is_map(id_to_params) do
     with {:ok, traces} <- trace_replay_transaction_responses_to_traces(responses, id_to_params) do

apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/variant.ex

@@ -13,6 +13,22 @@ defmodule EthereumJSONRPC.Variant do
 
   @type internal_transaction_params :: map()
 
+  @doc """
+  Fetch the block reward contract beneficiaries for a given block
+  range, from the variant of the Ethereum JSONRPC API.
+
+  For more information on block reward contracts see:
+  https://wiki.parity.io/Block-Reward-Contract.html
+
+  ## Returns
+
+   * `{:ok, #MapSet<[%{...}]>}` - beneficiaries were successfully fetched
+   * `{:error, reason}` - there was one or more errors with `reason` in fetching the beneficiaries
+   * `:ignore` - the variant does not support fetching beneficiaries
+  """
+  @callback fetch_beneficiaries(Range.t(), EthereumJSONRPC.json_rpc_named_arguments()) ::
+              {:ok, MapSet.t()} | {:error, reason :: term} | :ignore
+
   @doc """
   Fetches the `t:Explorer.Chain.InternalTransaction.changeset/2` params from the variant of the Ethereum JSONRPC API.
 

apps/ethereum_jsonrpc/test/ethereum_jsonrpc/parity_test.exs

@@ -222,4 +222,278 @@ defmodule EthereumJSONRPC.ParityTest do
                 ]}
     end
   end
+
+  describe "fetch_beneficiaries/1" do
+    test "with valid block range, returns {:ok, addresses}", %{
+      json_rpc_named_arguments: json_rpc_named_arguments
+    } do
+      block_number = 5_080_887
+      block_quantity = EthereumJSONRPC.integer_to_quantity(block_number)
+      hash1 = "0xef481b4e2c3ed62265617f2e9dfcdf3cf3efc11a"
+      hash2 = "0x523b6539ff08d72a6c8bb598af95bf50c1ea839c"
+
+      if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do
+        expect(EthereumJSONRPC.Mox, :json_rpc, fn %{params: [^block_quantity]}, _options ->
+          {:ok,
+           [
+             %{
+               "action" => %{
+                 "author" => hash1,
+                 "rewardType" => "block",
+                 "value" => "0xde0b6b3a7640000"
+               },
+               "blockHash" => "0x52a8d2185282506ce681364d2aa0c085ba45fdeb5d6c0ddec1131617a71ee2ca",
+               "blockNumber" => block_number,
+               "result" => nil,
+               "subtraces" => 0,
+               "traceAddress" => [],
+               "transactionHash" => nil,
+               "transactionPosition" => nil,
+               "type" => "reward"
+             },
+             %{
+               "action" => %{
+                 "author" => hash2,
+                 "rewardType" => "block",
+                 "value" => "0xde0b6b3a7640000"
+               },
+               "blockHash" => "0x52a8d2185282506ce681364d2aa0c085ba45fdeb5d6c0ddec1131617a71ee2ca",
+               "blockNumber" => block_number,
+               "result" => nil,
+               "subtraces" => 0,
+               "traceAddress" => [],
+               "transactionHash" => nil,
+               "transactionPosition" => nil,
+               "type" => "reward"
+             }
+           ]}
+        end)
+      end
+
+      expected_beneficiaries =
+        MapSet.new([
+          %{block_number: block_number, address_hash: hash2},
+          %{block_number: block_number, address_hash: hash1}
+        ])
+
+      {:ok, fetched_beneficiaries} =
+        EthereumJSONRPC.Parity.fetch_beneficiaries(5_080_887..5_080_887, json_rpc_named_arguments)
+
+      assert fetched_beneficiaries == expected_beneficiaries
+    end
+
+    test "with no rewards, returns {:ok, []}", %{
+      json_rpc_named_arguments: json_rpc_named_arguments
+    } do
+      if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do
+        expect(EthereumJSONRPC.Mox, :json_rpc, fn _json, _options ->
+          {:ok, []}
+        end)
+
+        {:ok, fetched_beneficiaries} =
+          EthereumJSONRPC.Parity.fetch_beneficiaries(5_080_887..5_080_887, json_rpc_named_arguments)
+
+        assert fetched_beneficiaries == MapSet.new()
+      end
+    end
+
+    test "with nil rewards, returns {:error, }", %{
+      json_rpc_named_arguments: json_rpc_named_arguments
+    } do
+      if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do
+        expect(EthereumJSONRPC.Mox, :json_rpc, fn _json, _options ->
+          {:ok, nil}
+        end)
+
+        result = EthereumJSONRPC.Parity.fetch_beneficiaries(5_080_887..5_080_887, json_rpc_named_arguments)
+
+        assert result == {:error, "Error fetching block reward contract beneficiaries"}
+      end
+    end
+
+    test "ignores non-reward traces", %{
+      json_rpc_named_arguments: json_rpc_named_arguments
+    } do
+      block_number = 5_077_429
+      block_quantity = EthereumJSONRPC.integer_to_quantity(block_number)
+      hash1 = "0xcfa53498686e00d3b4b41f3bea61604038eebb58"
+      hash2 = "0x523b6539ff08d72a6c8bb598af95bf50c1ea839c"
+
+      if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do
+        expect(EthereumJSONRPC.Mox, :json_rpc, fn %{params: [^block_quantity]}, _options ->
+          {:ok,
+           [
+             %{
+               "action" => %{
+                 "callType" => "call",
+                 "from" => "0x95426f2bc716022fcf1def006dbc4bb81f5b5164",
+                 "gas" => "0x0",
+                 "input" => "0x",
+                 "to" => "0xe797a1da01eb0f951e0e400f9343de9d17a06bac",
+                 "value" => "0x4a817c800"
+               },
+               "blockHash" => "0x6659a4926d833a7eab74379fa647ec74c9f5e65f8029552a35264126560f300a",
+               "blockNumber" => block_number,
+               "result" => %{"gasUsed" => "0x0", "output" => "0x"},
+               "subtraces" => 0,
+               "traceAddress" => [],
+               "transactionHash" => "0x5acf90f846b8216bdbc309cf4eb24adc69d730bf29304dc0e740cf6df850666e",
+               "transactionPosition" => 0,
+               "type" => "call"
+             },
+             %{
+               "action" => %{
+                 "author" => hash1,
+                 "rewardType" => "block",
+                 "value" => "0xde0b6b3a7640000"
+               },
+               "blockHash" => "0x6659a4926d833a7eab74379fa647ec74c9f5e65f8029552a35264126560f300a",
+               "blockNumber" => block_number,
+               "result" => nil,
+               "subtraces" => 0,
+               "traceAddress" => [],
+               "transactionHash" => nil,
+               "transactionPosition" => nil,
+               "type" => "reward"
+             },
+             %{
+               "action" => %{
+                 "author" => hash2,
+                 "rewardType" => "block",
+                 "value" => "0xde0b6b3a7640000"
+               },
+               "blockHash" => "0x6659a4926d833a7eab74379fa647ec74c9f5e65f8029552a35264126560f300a",
+               "blockNumber" => block_number,
+               "result" => nil,
+               "subtraces" => 0,
+               "traceAddress" => [],
+               "transactionHash" => nil,
+               "transactionPosition" => nil,
+               "type" => "reward"
+             }
+           ]}
+        end)
+      end
+
+      expected_beneficiaries =
+        MapSet.new([
+          %{block_number: block_number, address_hash: hash2},
+          %{block_number: block_number, address_hash: hash1}
+        ])
+
+      {:ok, fetched_beneficiaries} =
+        EthereumJSONRPC.Parity.fetch_beneficiaries(5_077_429..5_077_429, json_rpc_named_arguments)
+
+      assert fetched_beneficiaries == expected_beneficiaries
+    end
+
+    test "with multiple blocks with repeat beneficiaries", %{
+      json_rpc_named_arguments: json_rpc_named_arguments
+    } do
+      block_number1 = 5_080_886
+      block_quantity1 = EthereumJSONRPC.integer_to_quantity(block_number1)
+      block_number2 = 5_080_887
+      block_quantity2 = EthereumJSONRPC.integer_to_quantity(block_number2)
+      hash1 = "0xadc702c4bb09fbc502dd951856b9c7a1528a88de"
+      hash2 = "0xef481b4e2c3ed62265617f2e9dfcdf3cf3efc11a"
+      hash3 = "0x523b6539ff08d72a6c8bb598af95bf50c1ea839c"
+
+      if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do
+        expect(EthereumJSONRPC.Mox, :json_rpc, 2, fn
+          %{params: [^block_quantity1]} = _json, _options ->
+            {:ok,
+             [
+               %{
+                 "action" => %{
+                   "author" => hash1,
+                   "rewardType" => "block",
+                   "value" => "0xde0b6b3a7640000"
+                 },
+                 "blockNumber" => block_number1,
+                 "result" => nil,
+                 "subtraces" => 0,
+                 "traceAddress" => [],
+                 "transactionHash" => nil,
+                 "transactionPosition" => nil,
+                 "type" => "reward"
+               },
+               %{
+                 "action" => %{
+                   "author" => hash3,
+                   "rewardType" => "block",
+                   "value" => "0xde0b6b3a7640000"
+                 },
+                 "blockNumber" => block_number1,
+                 "result" => nil,
+                 "subtraces" => 0,
+                 "traceAddress" => [],
+                 "transactionHash" => nil,
+                 "transactionPosition" => nil,
+                 "type" => "reward"
+               }
+             ]}
+
+          %{params: [^block_quantity2]} = _json, _options ->
+            {:ok,
+             [
+               %{
+                 "action" => %{
+                   "author" => hash2,
+                   "rewardType" => "block",
+                   "value" => "0xde0b6b3a7640000"
+                 },
+                 "blockNumber" => block_number2,
+                 "result" => nil,
+                 "subtraces" => 0,
+                 "traceAddress" => [],
+                 "transactionHash" => nil,
+                 "transactionPosition" => nil,
+                 "type" => "reward"
+               },
+               %{
+                 "action" => %{
+                   "author" => hash3,
+                   "rewardType" => "block",
+                   "value" => "0xde0b6b3a7640000"
+                 },
+                 "blockNumber" => block_number2,
+                 "result" => nil,
+                 "subtraces" => 0,
+                 "traceAddress" => [],
+                 "transactionHash" => nil,
+                 "transactionPosition" => nil,
+                 "type" => "reward"
+               }
+             ]}
+        end)
+      end
+
+      expected_beneficiaries =
+        MapSet.new([
+          %{block_number: block_number1, address_hash: hash3},
+          %{block_number: block_number2, address_hash: hash3},
+          %{block_number: block_number2, address_hash: hash2},
+          %{block_number: block_number1, address_hash: hash1}
+        ])
+
+      {:ok, fetched_beneficiaries} =
+        EthereumJSONRPC.Parity.fetch_beneficiaries(5_080_886..5_080_887, json_rpc_named_arguments)
+
+      assert fetched_beneficiaries == expected_beneficiaries
+    end
+
+    test "with error, returns {:error, reason}", %{
+      json_rpc_named_arguments: json_rpc_named_arguments
+    } do
+      if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do
+        expect(EthereumJSONRPC.Mox, :json_rpc, fn _json, _options ->
+          {:error, "oops"}
+        end)
+
+        result = EthereumJSONRPC.Parity.fetch_beneficiaries(5_080_887..5_080_887, json_rpc_named_arguments)
+
+        assert result == {:error, "Error fetching block reward contract beneficiaries"}
+      end
+    end
+  end
 end

apps/ethereum_jsonrpc/test/ethereum_jsonrpc_test.exs

@@ -165,6 +165,19 @@ defmodule EthereumJSONRPCTest do
     end
   end
 
+  describe "fetch_beneficiaries/2" do
+    @tag :no_geth
+    test "fetches benefeciaries from variant API", %{json_rpc_named_arguments: json_rpc_named_arguments} do
+      if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do
+        expect(EthereumJSONRPC.Mox, :json_rpc, fn _, _ ->
+          {:ok, []}
+        end)
+
+        assert EthereumJSONRPC.fetch_beneficiaries(1..1, json_rpc_named_arguments) == {:ok, MapSet.new()}
+      end
+    end
+  end
+
   describe "fetch_block_by_hash/2" do
     test "can fetch blocks", %{json_rpc_named_arguments: json_rpc_named_arguments} do
       %{block_hash: block_hash, transaction_hash: transaction_hash} =

apps/indexer/config/dev/parity.exs

@@ -9,6 +9,7 @@ config :indexer,
       url: "https://sokol.poa.network",
       method_to_url: [
         eth_getBalance: "https://sokol-trace.poa.network",
+        trace_block: "https://sokol-trace.poa.network",
         trace_replayTransaction: "https://sokol-trace.poa.network"
       ],
       http_options: [recv_timeout: 60_000, timeout: 60_000, hackney: [pool: :ethereum_jsonrpc]]

apps/indexer/config/prod/parity.exs

@@ -9,6 +9,7 @@ config :indexer,
       url: System.get_env("ETHEREUM_URL") || "https://sokol.poa.network",
       method_to_url: [
         eth_getBalance: System.get_env("TRACE_URL") || "https://sokol-trace.poa.network",
+        trace_block: System.get_env("TRACE_URL") || "https://sokol-trace.poa.network",
         trace_replayTransaction: System.get_env("TRACE_URL") || "https://sokol-trace.poa.network"
       ],
       http_options: [recv_timeout: 60_000, timeout: 60_000, hackney: [pool: :ethereum_jsonrpc]]

apps/indexer/lib/indexer/address_extraction.ex

@@ -116,6 +116,12 @@ defmodule Indexer.AddressExtraction do
         %{from: :block_number, to: :fetched_coin_balance_block_number},
         %{from: :to_address_hash, to: :hash}
       ]
+    ],
+    block_reward_contract_beneficiaries: [
+      [
+        %{from: :block_number, to: :fetched_coin_balance_block_number},
+        %{from: :address_hash, to: :hash}
+      ]
     ]
   }
 
@@ -379,6 +385,12 @@ defmodule Indexer.AddressExtraction do
               required(:to_address_hash) => String.t(),
               required(:block_number) => non_neg_integer()
             }
+          ],
+          optional(:block_reward_contract_beneficiaries) => [
+            %{
+              required(:address_hash) => String.t(),
+              required(:block_number) => non_neg_integer()
+            }
           ]
         }) :: [params]
   def extract_addresses(fetched_data, options \\ []) when is_map(fetched_data) and is_list(options) do

apps/indexer/lib/indexer/block/fetcher.ex

@@ -101,8 +101,10 @@ defmodule Indexer.Block.Fetcher do
          transactions_with_receipts = Receipts.put(transactions_without_receipts, receipts),
          %{token_transfers: token_transfers, tokens: tokens} = TokenTransfers.parse(logs),
          %{mint_transfers: mint_transfers} = MintTransfer.parse(logs),
+         {:beneficiaries, {:ok, beneficiaries}} <- fetch_beneficiaries(range, json_rpc_named_arguments),
          addresses =
            AddressExtraction.extract_addresses(%{
+             block_reward_contract_beneficiaries: MapSet.to_list(beneficiaries),
              blocks: blocks,
              logs: logs,
              mint_transfers: mint_transfers,
@@ -110,11 +112,13 @@ defmodule Indexer.Block.Fetcher do
              transactions: transactions_with_receipts
            }),
          coin_balances_params_set =
-           CoinBalances.params_set(%{
+           %{
              blocks_params: blocks,
              logs_params: logs,
              transactions_params: transactions_with_receipts
-           }),
+           }
+           |> CoinBalances.params_set()
+           |> MapSet.union(beneficiaries),
          address_token_balances = TokenBalances.params_set(%{token_transfers_params: token_transfers}),
          {:ok, inserted} <-
            __MODULE__.import(
@@ -191,6 +195,16 @@ defmodule Indexer.Block.Fetcher do
 
   def async_import_uncles(_), do: :ok
 
+  defp fetch_beneficiaries(range, json_rpc_named_arguments) do
+    result =
+      case EthereumJSONRPC.fetch_beneficiaries(range, json_rpc_named_arguments) do
+        :ignore -> {:ok, MapSet.new()}
+        result -> result
+      end
+
+    {:beneficiaries, result}
+  end
+
   # `fetched_balance_block_number` is needed for the `CoinBalanceFetcher`, but should not be used for `import` because the
   # balance is not known yet.
   defp pop_address_hash_to_fetched_balance_block_number(options) do

apps/indexer/test/indexer/address_extraction_test.exs

@@ -107,12 +107,18 @@ defmodule Indexer.AddressExtractionTest do
         token_contract_address_hash: gen_hash()
       }
 
+      beneficiary = %{
+        block_number: 6,
+        address_hash: gen_hash()
+      }
+
       blockchain_data = %{
         blocks: [block],
         internal_transactions: [internal_transaction],
         transactions: [transaction],
         logs: [log],
-        token_transfers: [token_transfer]
+        token_transfers: [token_transfer],
+        block_reward_contract_beneficiaries: [beneficiary]
       }
 
       assert AddressExtraction.extract_addresses(blockchain_data) == [
@@ -144,6 +150,10 @@ defmodule Indexer.AddressExtractionTest do
                %{
                  hash: token_transfer.token_contract_address_hash,
                  fetched_coin_balance_block_number: token_transfer.block_number
+               },
+               %{
+                 hash: beneficiary.address_hash,
+                 fetched_coin_balance_block_number: beneficiary.block_number
                }
              ]
     end

apps/indexer/test/indexer/block/catchup/bound_interval_supervisor_test.exs

@@ -63,6 +63,9 @@ defmodule Indexer.Block.Catchup.BoundIntervalSupervisorTest do
                    "uncles" => []
                  }}
 
+              %{method: "trace_block"}, _options ->
+                {:ok, []}
+
               [%{method: "eth_getBlockByNumber", params: [_, true]} | _] = requests, _options ->
                 {:ok,
                  Enum.map(requests, fn %{id: id, params: [block_quantity, true]} ->
@@ -464,6 +467,17 @@ defmodule Indexer.Block.Catchup.BoundIntervalSupervisorTest do
            }
          ]}
       end)
+      |> (fn mock ->
+            case Keyword.fetch!(json_rpc_named_arguments, :variant) do
+              EthereumJSONRPC.Parity ->
+                expect(mock, :json_rpc, fn %{method: "trace_block"}, _options ->
+                  {:ok, []}
+                end)
+
+              _ ->
+                mock
+            end
+          end).()
       |> stub(:json_rpc, fn [
                               %{
                                 id: id,

apps/indexer/test/indexer/block/fetcher_test.exs

@@ -109,6 +109,9 @@ defmodule Indexer.Block.FetcherTest do
                  }
                ]}
             end)
+            |> expect(:json_rpc, fn %{id: _id, method: "trace_block", params: [^block_quantity]}, _options ->
+              {:ok, []}
+            end)
             |> expect(:json_rpc, fn [
                                       %{
                                         id: id,
@@ -356,6 +359,10 @@ defmodule Indexer.Block.FetcherTest do
                  }
                ]}
             end)
+            |> expect(:json_rpc, fn json, _options ->
+              assert %{id: _id, method: "trace_block", params: [^block_quantity]} = json
+              {:ok, []}
+            end)
             # async requests need to be grouped in one expect because the order is non-deterministic while multiple expect
             # calls on the same name/arity are used in order
             |> expect(:json_rpc, 5, fn json, _options ->

apps/indexer/test/indexer/block/realtime/fetcher_test.exs

@@ -24,7 +24,8 @@ defmodule Indexer.Block.Realtime.FetcherTest do
       |> put_in(
         [:transport_options, :method_to_url],
         eth_getBalance: "https://core-trace.poa.network",
-        trace_replayTransaction: "https://core-trace.poa.network"
+        trace_replayTransaction: "https://core-trace.poa.network",
+        trace_block: "https://core-trace.poa.network"
       )
 
     block_fetcher = %Indexer.Block.Fetcher{
@@ -195,6 +196,9 @@ defmodule Indexer.Block.Realtime.FetcherTest do
              }
            ]}
         end)
+        |> expect(:json_rpc, 2, fn %{method: "trace_block"}, _options ->
+          {:ok, []}
+        end)
         |> expect(:json_rpc, fn [
                                   %{
                                     id: 0,