Merge branch 'master' into 20181103-873-improve-top-accounts-listing-performance

.circleci/config.yml

@@ -14,6 +14,8 @@ jobs:
     working_directory: ~/app
 
     steps:
+      - run: sudo apt-get update; sudo apt-get -y install autoconf build-essential libgmp3-dev libtool
+
       - checkout
 
       - run: mix local.hex --force
@@ -70,6 +72,11 @@ jobs:
 
       - run: mix compile
 
+      # Ensure NIF is compiled for libsecp256k1
+      - run:
+          command: make
+          working_directory: "deps/libsecp256k1"
+
       # `deps` needs to be cached with `_build` because `_build` will symlink into `deps`
 
       - save_cache:

apps/block_scout_web/lib/block_scout_web/templates/smart_contract/_functions.html.eex

@@ -45,13 +45,13 @@
                 <div data-wei-ether-converter>
                   <span data-conversion-unit><%= output["value"] %></span>
                   <span class="py-2 px-2">
-                    <input class="wei-ether" type="checkbox" autocomplete="off"> 
+                    <input class="wei-ether" type="checkbox" autocomplete="off">
                     <span class="d-inline-block" data-conversion-text-wei><%= gettext("WEI")%></span>
                     <span class="d-none" data-conversion-text-eth><%= gettext("ETH")%></span>
                   </span>
                 </div>
               <% else %>
-                <%= output["value"] %>
+                <%= values(output["value"], output["type"]) %>
               <% end %>
           <% end %>
         <% end %>

apps/block_scout_web/lib/block_scout_web/views/smart_contract_view.ex

@@ -10,6 +10,12 @@ defmodule BlockScoutWeb.SmartContractView do
   def named_argument?(%{"name" => _}), do: true
   def named_argument?(_), do: false
 
+  def values(addresses, type) when type == "address[]" do
+    addresses
+    |> Enum.map(&values(&1, "address"))
+    |> Enum.join(", ")
+  end
+
   def values(value, type) when type in ["address", "address payable"] do
     {:ok, address} = Explorer.Chain.Hash.Address.cast(value)
     to_string(address)

apps/block_scout_web/test/block_scout_web/views/tokens/smart_contract_view_test.exs

@@ -80,6 +80,16 @@ defmodule BlockScoutWeb.SmartContractViewTest do
       assert SmartContractView.values(value, "address payable") == "0x5f26097334b6a32b7951df61fd0c5803ec5d8354"
     end
 
+    test "convert each value to string and join them when receiving 'address[]' as the type" do
+      value = [
+        <<95, 38, 9, 115, 52, 182, 163, 43, 121, 81, 223, 97, 253, 12, 88, 3, 236, 93, 131, 84>>,
+        <<207, 38, 14, 163, 23, 85, 86, 55, 197, 95, 112, 229, 93, 186, 141, 90, 216, 65, 76, 176>>
+      ]
+
+      assert SmartContractView.values(value, "address[]") ==
+               "0x5f26097334b6a32b7951df61fd0c5803ec5d8354, 0xcf260ea317555637c55f70e55dba8d5ad8414cb0"
+    end
+
     test "returns the value when the type is neither 'address' nor 'address payable'" do
       value = "POA"
 

apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/block.ex

@@ -11,16 +11,23 @@ defmodule EthereumJSONRPC.Block do
   @type elixir :: %{String.t() => non_neg_integer | DateTime.t() | String.t() | nil}
   @type params :: %{
           difficulty: pos_integer(),
+          extra_data: EthereumJSONRPC.hash(),
           gas_limit: non_neg_integer(),
           gas_used: non_neg_integer(),
           hash: EthereumJSONRPC.hash(),
+          logs_bloom: EthereumJSONRPC.hash(),
           miner_hash: EthereumJSONRPC.hash(),
+          mix_hash: EthereumJSONRPC.hash(),
           nonce: EthereumJSONRPC.hash(),
           number: non_neg_integer(),
           parent_hash: EthereumJSONRPC.hash(),
+          receipts_root: EthereumJSONRPC.hash(),
+          sha3_uncles: EthereumJSONRPC.hash(),
           size: non_neg_integer(),
+          state_root: EthereumJSONRPC.hash(),
           timestamp: DateTime.t(),
           total_difficulty: non_neg_integer(),
+          transactions_root: EthereumJSONRPC.hash(),
           uncles: [EthereumJSONRPC.hash()]
         }
 
@@ -95,16 +102,23 @@ defmodule EthereumJSONRPC.Block do
       ...> )
       %{
         difficulty: 340282366920938463463374607431465537093,
+        extra_data: "0xd5830108048650617269747986312e32322e31826c69",
         gas_limit: 6706541,
         gas_used: 0,
         hash: "0x52c867bc0a91e573dc39300143c3bead7408d09d45bdb686749f02684ece72f3",
+        logs_bloom: "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
         miner_hash: "0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca",
+        mix_hash: "0x0",
         nonce: 0,
         number: 1,
         parent_hash: "0x5b28c1bfd3a15230c9a46b399cd0f9a6920d432e85381cc6a140b06e8410112f",
+        receipts_root: "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
+        sha3_uncles: "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347",
         size: 576,
+        state_root: "0xc196ad59d867542ef20b29df5f418d07dc7234f4bc3d25260526620b7958a8fb",
         timestamp: Timex.parse!("2017-12-15T21:03:30Z", "{ISO:Extended:Z}"),
         total_difficulty: 340282366920938463463374607431465668165,
+        transactions_root: "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
         uncles: []
       }
 
@@ -136,16 +150,23 @@ defmodule EthereumJSONRPC.Block do
       ...> )
       %{
         difficulty: 17561410778,
+        extra_data: "0x476574682f4c5649562f76312e302e302f6c696e75782f676f312e342e32",
         gas_limit: 5000,
         gas_used: 0,
         hash: "0x4d9423080290a650eaf6db19c87c76dff83d1b4ab64aefe6e5c5aa2d1f4b6623",
+        logs_bloom: "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+        mix_hash: "0xbbb93d610b2b0296a59f18474ac3d6086a9902aa7ca4b9a306692f7c3d496fdf",
         miner_hash: "0xbb7b8287f3f0a933474a79eae42cbca977791171",
         nonce: 5539500215739777653,
         number: 59,
         parent_hash: "0xcd5b5c4cecd7f18a13fe974255badffd58e737dc67596d56bc01f063dd282e9e",
+        receipts_root: "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
+        sha3_uncles: "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347",
         size: 542,
+        state_root: "0x6fd0a5d82ca77d9f38c3ebbde11b11d304a5fcf3854f291df64395ab38ed43ba",
         timestamp: Timex.parse!("2015-07-30T15:32:07Z", "{ISO:Extended:Z}"),
         total_difficulty: 1039309006117,
+        transactions_root: "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
         uncles: []
       }
 
@@ -154,30 +175,43 @@ defmodule EthereumJSONRPC.Block do
   def elixir_to_params(
         %{
           "difficulty" => difficulty,
+          "extraData" => extra_data,
           "gasLimit" => gas_limit,
           "gasUsed" => gas_used,
           "hash" => hash,
+          "logsBloom" => logs_bloom,
           "miner" => miner_hash,
           "number" => number,
           "parentHash" => parent_hash,
+          "receiptsRoot" => receipts_root,
+          "sha3Uncles" => sha3_uncles,
           "size" => size,
+          "stateRoot" => state_root,
           "timestamp" => timestamp,
           "totalDifficulty" => total_difficulty,
+          "transactionsRoot" => transactions_root,
           "uncles" => uncles
         } = elixir
       ) do
     %{
       difficulty: difficulty,
+      extra_data: extra_data,
       gas_limit: gas_limit,
       gas_used: gas_used,
       hash: hash,
+      logs_bloom: logs_bloom,
       miner_hash: miner_hash,
+      mix_hash: Map.get(elixir, "mixHash", "0x0"),
       nonce: Map.get(elixir, "nonce", 0),
       number: number,
       parent_hash: parent_hash,
+      receipts_root: receipts_root,
+      sha3_uncles: sha3_uncles,
       size: size,
+      state_root: state_root,
       timestamp: timestamp,
       total_difficulty: total_difficulty,
+      transactions_root: transactions_root,
       uncles: uncles
     }
   end

apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/blocks.ex

@@ -45,16 +45,23 @@ defmodule EthereumJSONRPC.Blocks do
       [
         %{
           difficulty: 131072,
+          extra_data: "0x",
           gas_limit: 6700000,
           gas_used: 0,
           hash: "0x5b28c1bfd3a15230c9a46b399cd0f9a6920d432e85381cc6a140b06e8410112f",
+          logs_bloom: "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
           miner_hash: "0x0000000000000000000000000000000000000000",
+          mix_hash: "0x0",
           nonce: 0,
           number: 0,
           parent_hash: "0x0000000000000000000000000000000000000000000000000000000000000000",
+          receipts_root: "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
+          sha3_uncles: "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347",
           size: 533,
+          state_root: "0xfad4af258fd11939fae0c6c6eec9d340b1caac0b0196fd9a1bc3f489c5bf00b3",
           timestamp: Timex.parse!("1970-01-01T00:00:00Z", "{ISO:Extended:Z}"),
           total_difficulty: 131072,
+          transactions_root: "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
           uncles: ["0xe670ec64341771606e55d6b4ca35a1a6b75ee3d5145a99d05921026d15273311"]
         }
       ]

apps/ethereum_jsonrpc/test/ethereum_jsonrpc/request_coordinator_test.exs

@@ -10,15 +10,6 @@ defmodule EthereumJSONRPC.RequestCoordinatorTest do
   setup :set_mox_global
   setup :verify_on_exit!
 
-  defp sleep_time(timeouts) do
-    wait_per_timeout =
-      :ethereum_jsonrpc
-      |> Application.get_env(RequestCoordinator)
-      |> Keyword.fetch!(:wait_per_timeout)
-
-    timeouts * wait_per_timeout
-  end
-
   setup do
     table = Application.get_env(:ethereum_jsonrpc, EthereumJSONRPC.RequestCoordinator)[:rolling_window_opts][:table]
 

apps/ethereum_jsonrpc/test/ethereum_jsonrpc/web_socket/web_socket_client_test.exs

@@ -95,18 +95,6 @@ defmodule EthereumJSONRPC.WebSocket.WebSocketClientTest do
     end
   end
 
-  defp cowboy(0) do
-    dispatch = :cowboy_router.compile([{:_, [{"/websocket", EthereumJSONRPC.WebSocket.Cowboy.WebSocketHandler, []}]}])
-    {:ok, _} = :cowboy.start_http(EthereumJSONRPC.WebSocket.Cowboy, 100, [], env: [dispatch: dispatch])
-    :ranch.get_port(EthereumJSONRPC.WebSocket.Cowboy)
-  end
-
-  defp cowboy(port) do
-    dispatch = :cowboy_router.compile([{:_, [{"/websocket", EthereumJSONRPC.WebSocket.Cowboy.WebSocketHandler, []}]}])
-    {:ok, _} = :cowboy.start_http(EthereumJSONRPC.WebSocket.Cowboy, 100, [port: port], env: [dispatch: dispatch])
-    port
-  end
-
   defp example_state(_) do
     %{state: %WebSocketClient{url: "ws://example.com"}}
   end

apps/ethereum_jsonrpc/test/ethereum_jsonrpc_test.exs

@@ -207,10 +207,15 @@ defmodule EthereumJSONRPCTest do
                  "gasLimit" => "0x0",
                  "gasUsed" => "0x0",
                  "hash" => block_hash,
+                 "extraData" => "0x0",
+                 "logsBloom" => "0x0",
                  "miner" => "0x0",
                  "number" => block_number,
                  "parentHash" => "0x0",
+                 "receiptsRoot" => "0x0",
                  "size" => "0x0",
+                 "sha3Uncles" => "0x0",
+                 "stateRoot" => "0x0",
                  "timestamp" => "0x0",
                  "totalDifficulty" => "0x0",
                  "transactions" => [
@@ -231,6 +236,7 @@ defmodule EthereumJSONRPCTest do
                      "value" => "0x0"
                    }
                  ],
+                 "transactionsRoot" => "0x0",
                  "uncles" => []
                }
              }
@@ -364,16 +370,22 @@ defmodule EthereumJSONRPCTest do
              id: 0,
              result: %{
                "difficulty" => "0x0",
+               "extraData" => "0x0",
                "gasLimit" => "0x0",
                "gasUsed" => "0x0",
                "hash" => "0x0",
+               "logsBloom" => "0x0",
                "miner" => "0x0",
                "number" => "0x0",
                "parentHash" => "0x0",
+               "receiptsRoot" => "0x0",
+               "sha3Uncles" => "0x0",
                "size" => "0x0",
+               "stateRoot" => "0x0",
                "timestamp" => "0x0",
                "totalDifficulty" => "0x0",
                "transactions" => [],
+               "transactionsRoot" => "0x0",
                "uncles" => []
              },
              jsonrpc: "2.0"

apps/indexer/config/config.exs

@@ -5,9 +5,10 @@ use Mix.Config
 import Bitwise
 
 config :indexer,
+  block_transformer: Indexer.Block.Transform.Base,
+  ecto_repos: [Explorer.Repo],
   # bytes
-  memory_limit: 1 <<< 30,
+  memory_limit: 1 <<< 30
-  ecto_repos: [Explorer.Repo]
 
 config :logger, :indexer,
   # keep synced with `config/config.exs`

apps/indexer/lib/indexer/block/fetcher.ex

@@ -9,6 +9,7 @@ defmodule Indexer.Block.Fetcher do
   alias Indexer.{AddressExtraction, CoinBalance, MintTransfer, Token, TokenTransfers}
   alias Indexer.Address.{CoinBalances, TokenBalances}
   alias Indexer.Block.Fetcher.Receipts
+  alias Indexer.Block.Transform
 
   @type address_hash_to_fetched_balance_block_number :: %{String.t() => Block.block_number()}
   @type transaction_hash_to_block_number :: %{String.t() => Block.block_number()}
@@ -96,6 +97,7 @@ defmodule Indexer.Block.Fetcher do
            transactions: transactions_without_receipts,
            block_second_degree_relations: block_second_degree_relations
          } = result,
+         blocks = Transform.transform_blocks(blocks),
          {:receipts, {:ok, receipt_params}} <- {:receipts, Receipts.fetch(state, transactions_without_receipts)},
          %{logs: logs, receipts: receipts} = receipt_params,
          transactions_with_receipts = Receipts.put(transactions_without_receipts, receipts),

apps/indexer/lib/indexer/block/transform.ex

@@ -0,0 +1,31 @@
+defmodule Indexer.Block.Transform do
+  @moduledoc """
+  Protocol for transforming blocks.
+  """
+
+  @type block :: map()
+
+  @doc """
+  Transforms a block.
+  """
+  @callback transform(block :: block()) :: block()
+
+  @doc """
+  Runs a list of blocks through the configured block transformer.
+  """
+  def transform_blocks(blocks) when is_list(blocks) do
+    transformer = Application.get_env(:indexer, :block_transformer)
+
+    unless transformer do
+      raise ArgumentError,
+            """
+            No block transformer defined. Set a blocker transformer."
+
+               config :indexer,
+                 block_transformer: Indexer.Block.Transform.Base
+            """
+    end
+
+    Enum.map(blocks, &transformer.transform/1)
+  end
+end

apps/indexer/lib/indexer/block/transform/base.ex

@@ -0,0 +1,14 @@
+defmodule Indexer.Block.Transform.Base do
+  @moduledoc """
+  Default block transformer to be used.
+  """
+
+  alias Indexer.Block.Transform
+
+  @behaviour Transform
+
+  @impl Transform
+  def transform(block) when is_map(block) do
+    block
+  end
+end

apps/indexer/lib/indexer/block/transform/clique.ex

@@ -0,0 +1,16 @@
+defmodule Indexer.Block.Transform.Clique do
+  @moduledoc """
+  Handles block transforms for Clique chain.
+  """
+
+  alias Indexer.Block.{Transform, Util}
+
+  @behaviour Transform
+
+  @impl Transform
+  def transform(block) when is_map(block) do
+    miner_address = Util.signer(block)
+
+    %{block | miner_hash: miner_address}
+  end
+end

apps/indexer/lib/indexer/block/util.ex

@@ -0,0 +1,75 @@
+defmodule Indexer.Block.Util do
+  @moduledoc """
+  Helper functions for parsing block information.
+  """
+
+  @doc """
+  Calculates the signer's address by recovering the ECDSA public key.
+
+  https://en.wikipedia.org/wiki/Elliptic_Curve_Digital_Signature_Algorithm
+  """
+  def signer(block) when is_map(block) do
+    # Last 65 bytes is the signature. Multiply by two since we haven't transformed to raw bytes
+    {extra_data, signature} = String.split_at(trim_prefix(block.extra_data), -130)
+
+    block = %{block | extra_data: extra_data}
+
+    signature_hash = signature_hash(block)
+
+    recover_pub_key(signature_hash, decode(signature))
+  end
+
+  # Signature hash calculated from the block header.
+  # Needed for PoA-based chains
+  defp signature_hash(block) do
+    header_data = [
+      decode(block.parent_hash),
+      decode(block.sha3_uncles),
+      decode(block.miner_hash),
+      decode(block.state_root),
+      decode(block.transactions_root),
+      decode(block.receipts_root),
+      decode(block.logs_bloom),
+      block.difficulty,
+      block.number,
+      block.gas_limit,
+      block.gas_used,
+      DateTime.to_unix(block.timestamp),
+      decode(block.extra_data),
+      decode(block.mix_hash),
+      decode(block.nonce)
+    ]
+
+    :keccakf1600.hash(:sha3_256, ExRLP.encode(header_data))
+  end
+
+  defp trim_prefix("0x" <> rest), do: rest
+
+  defp decode("0x" <> rest) do
+    decode(rest)
+  end
+
+  defp decode(data) do
+    Base.decode16!(data, case: :mixed)
+  end
+
+  # Recovers the key from the signature hash and signature
+  defp recover_pub_key(signature_hash, signature) do
+    <<
+      r::bytes-size(32),
+      s::bytes-size(32),
+      v::integer-size(8)
+    >> = signature
+
+    # First byte represents compression which can be ignored
+    # Private key is the last 64 bytes
+    {:ok, <<_compression::bytes-size(1), private_key::binary>>} =
+      :libsecp256k1.ecdsa_recover_compact(signature_hash, r <> s, :uncompressed, v)
+
+    # Public key comes from the last 20 bytes
+    <<_::bytes-size(12), public_key::binary>> = :keccakf1600.hash(:sha3_256, private_key)
+
+    miner_address = Base.encode16(public_key, case: :lower)
+    "0x" <> miner_address
+  end
+end

apps/indexer/mix.exs

@@ -46,10 +46,14 @@ defmodule Indexer.MixProject do
     [
       # JSONRPC access to Parity for `Explorer.Indexer`
       {:ethereum_jsonrpc, in_umbrella: true},
+      # RLP encoding
+      {:ex_rlp, "~> 0.3"},
       # Code coverage
       {:excoveralls, "~> 0.10.0", only: [:test], github: "KronicDeth/excoveralls", branch: "circle-workflows"},
       # Importing to database
       {:explorer, in_umbrella: true},
+      # libsecp2561k1 crypto functions
+      {:libsecp256k1, "~> 0.1.10"},
       # Log errors and application output to separate files
       {:logger_file_backend, "~> 0.0.10"},
       # Mocking `EthereumJSONRPC.Transport`, so we avoid hitting real chains for local testing

apps/indexer/test/indexer/block/catchup/bound_interval_supervisor_test.exs

@@ -448,20 +448,26 @@ defmodule Indexer.Block.Catchup.BoundIntervalSupervisorTest do
              jsonrpc: "2.0",
              result: %{
                "difficulty" => "0x0",
+               "extraData" => "0x0",
                "gasLimit" => "0x0",
                "gasUsed" => "0x0",
                "hash" =>
                  Explorer.Factory.block_hash()
                  |> to_string(),
+               "logsBloom" => "0x0",
                "miner" => "0xb2930b35844a230f00e51431acae96fe543a0347",
                "number" => "0x0",
                "parentHash" =>
                  Explorer.Factory.block_hash()
                  |> to_string(),
+               "receiptsRoot" => "0x0",
+               "sha3Uncles" => "0x0",
                "size" => "0x0",
+               "stateRoot" => "0x0",
                "timestamp" => "0x0",
                "totalDifficulty" => "0x0",
                "transactions" => [],
+               "transactionsRoot" => "0x0",
                "uncles" => []
              }
            }

apps/indexer/test/indexer/block/transform/base_test.exs

@@ -0,0 +1,42 @@
+defmodule Indexer.Block.Transform.BaseTest do
+  use ExUnit.Case
+
+  alias Indexer.Block.Transform.Base
+
+  @block %{
+    difficulty: 1,
+    extra_data:
+      "0xd68301080d846765746886676f312e3130856c696e7578000000000000000000773ab2ca8f47904a14739ad80a75b71d9d29b9fff8b7ecdcb73efffa6f74122f17d304b5dc8e6e5f256c9474dd115c8d4dae31b7a3d409e5c3270f8fde41cd8c00",
+    gas_limit: 7_753_377,
+    gas_used: 1_810_195,
+    hash: "0x7004c895e812c55b0c2be8a46d72ca300a683dc27d1d7917ee7742d4d0359c1f",
+    logs_bloom:
+      "0x00000000000000020000000000002000000400000000000000000000000000000000000000000000040000080004000020000010000000000000000000000000000000000000000008000008000000000000000000200000000000000000000000000000020000000000000000000800000000000000804000000010080000000800000000000000000000000000000000000000000000800000000000080000000008000400000000404000000000000000000000000200000000000000000000000002000000000000001002000000000000002000000008000000000020000000000000000000000000000000000000000000000000400000800000000000",
+    miner: "0x0000000000000000000000000000000000000000",
+    mix_hash: "0x0000000000000000000000000000000000000000000000000000000000000000",
+    nonce: "0x0000000000000000",
+    number: 2_848_394,
+    parent_hash: "0x20350fc367e19d3865be1ea7da72ab81f8f9941c43ac6bb24a34a0a7caa2f3df",
+    receipts_root: "0x6ade4ac1079ea50cfadcce2b75ffbe4f9b14bf69b4607bbf1739463076ca6246",
+    sha3_uncles: "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347",
+    size: 6437,
+    state_root: "0x23f63347851bcd109059d007d71e19c4f5e73b7f0862bebcd04458333a004d92",
+    timestamp: DateTime.from_unix!(1_534_796_040),
+    total_difficulty: 5_353_647,
+    transactions: [
+      "0x7e3bb851fc74a436826d2af6b96e4db9484431811ef0d9c9e78370488d33d4e5",
+      "0x3976fd1e3d2a715c3cfcfde9bd3210798c26c017b8edb841d319227ecb3322fb",
+      "0xd8db124005bb8b6fda7b71fd56ac782552a66af58fe843ba3c4930423b87d1d2",
+      "0x10c1a1ca4d9f4b2bd5b89f7bbcbbc2d69e166fe23662b8db4f6beae0f50ac9fd",
+      "0xaa58a6545677c796a56b8bc874174c8cfd31a6c6e6ca3a87e086d4f66d52858a"
+    ],
+    transactions_root: "0xde8d25c0b9b54310128a21601331094b43f910f9f96102869c2e2dca94884bf4",
+    uncles: []
+  }
+
+  describe "transform/1" do
+    test "passes the block through unchanged" do
+      assert Base.transform(@block) == @block
+    end
+  end
+end

apps/indexer/test/indexer/block/transform/clique_test.exs

@@ -0,0 +1,43 @@
+defmodule Indexer.Block.Transform.CliqueTest do
+  use ExUnit.Case
+
+  alias Indexer.Block.Transform.Clique
+
+  @block %{
+    difficulty: 1,
+    extra_data:
+      "0xd68301080d846765746886676f312e3130856c696e7578000000000000000000773ab2ca8f47904a14739ad80a75b71d9d29b9fff8b7ecdcb73efffa6f74122f17d304b5dc8e6e5f256c9474dd115c8d4dae31b7a3d409e5c3270f8fde41cd8c00",
+    gas_limit: 7_753_377,
+    gas_used: 1_810_195,
+    hash: "0x7004c895e812c55b0c2be8a46d72ca300a683dc27d1d7917ee7742d4d0359c1f",
+    logs_bloom:
+      "0x00000000000000020000000000002000000400000000000000000000000000000000000000000000040000080004000020000010000000000000000000000000000000000000000008000008000000000000000000200000000000000000000000000000020000000000000000000800000000000000804000000010080000000800000000000000000000000000000000000000000000800000000000080000000008000400000000404000000000000000000000000200000000000000000000000002000000000000001002000000000000002000000008000000000020000000000000000000000000000000000000000000000000400000800000000000",
+    miner_hash: "0x0000000000000000000000000000000000000000",
+    mix_hash: "0x0000000000000000000000000000000000000000000000000000000000000000",
+    nonce: "0x0000000000000000",
+    number: 2_848_394,
+    parent_hash: "0x20350fc367e19d3865be1ea7da72ab81f8f9941c43ac6bb24a34a0a7caa2f3df",
+    receipts_root: "0x6ade4ac1079ea50cfadcce2b75ffbe4f9b14bf69b4607bbf1739463076ca6246",
+    sha3_uncles: "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347",
+    size: 6437,
+    state_root: "0x23f63347851bcd109059d007d71e19c4f5e73b7f0862bebcd04458333a004d92",
+    timestamp: DateTime.from_unix!(1_534_796_040),
+    total_difficulty: 5_353_647,
+    transactions: [
+      "0x7e3bb851fc74a436826d2af6b96e4db9484431811ef0d9c9e78370488d33d4e5",
+      "0x3976fd1e3d2a715c3cfcfde9bd3210798c26c017b8edb841d319227ecb3322fb",
+      "0xd8db124005bb8b6fda7b71fd56ac782552a66af58fe843ba3c4930423b87d1d2",
+      "0x10c1a1ca4d9f4b2bd5b89f7bbcbbc2d69e166fe23662b8db4f6beae0f50ac9fd",
+      "0xaa58a6545677c796a56b8bc874174c8cfd31a6c6e6ca3a87e086d4f66d52858a"
+    ],
+    transactions_root: "0xde8d25c0b9b54310128a21601331094b43f910f9f96102869c2e2dca94884bf4",
+    uncles: []
+  }
+
+  describe "transform/1" do
+    test "updates the miner hash with signer address" do
+      expected = %{@block | miner_hash: "0xfc18cbc391de84dbd87db83b20935d3e89f5dd91"}
+      assert Clique.transform(@block) == expected
+    end
+  end
+end

apps/indexer/test/indexer/block/transform_test.exs

@@ -0,0 +1,56 @@
+defmodule Indexer.Block.TransformTest do
+  use ExUnit.Case
+
+  alias Indexer.Block.Transform
+
+  @block %{
+    difficulty: 1,
+    extra_data:
+      "0xd68301080d846765746886676f312e3130856c696e7578000000000000000000773ab2ca8f47904a14739ad80a75b71d9d29b9fff8b7ecdcb73efffa6f74122f17d304b5dc8e6e5f256c9474dd115c8d4dae31b7a3d409e5c3270f8fde41cd8c00",
+    gas_limit: 7_753_377,
+    gas_used: 1_810_195,
+    hash: "0x7004c895e812c55b0c2be8a46d72ca300a683dc27d1d7917ee7742d4d0359c1f",
+    logs_bloom:
+      "0x00000000000000020000000000002000000400000000000000000000000000000000000000000000040000080004000020000010000000000000000000000000000000000000000008000008000000000000000000200000000000000000000000000000020000000000000000000800000000000000804000000010080000000800000000000000000000000000000000000000000000800000000000080000000008000400000000404000000000000000000000000200000000000000000000000002000000000000001002000000000000002000000008000000000020000000000000000000000000000000000000000000000000400000800000000000",
+    miner_hash: "0x0000000000000000000000000000000000000000",
+    mix_hash: "0x0000000000000000000000000000000000000000000000000000000000000000",
+    nonce: "0x0000000000000000",
+    number: 2_848_394,
+    parent_hash: "0x20350fc367e19d3865be1ea7da72ab81f8f9941c43ac6bb24a34a0a7caa2f3df",
+    receipts_root: "0x6ade4ac1079ea50cfadcce2b75ffbe4f9b14bf69b4607bbf1739463076ca6246",
+    sha3_uncles: "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347",
+    size: 6437,
+    state_root: "0x23f63347851bcd109059d007d71e19c4f5e73b7f0862bebcd04458333a004d92",
+    timestamp: DateTime.from_unix!(1_534_796_040),
+    total_difficulty: 5_353_647,
+    transactions: [
+      "0x7e3bb851fc74a436826d2af6b96e4db9484431811ef0d9c9e78370488d33d4e5",
+      "0x3976fd1e3d2a715c3cfcfde9bd3210798c26c017b8edb841d319227ecb3322fb",
+      "0xd8db124005bb8b6fda7b71fd56ac782552a66af58fe843ba3c4930423b87d1d2",
+      "0x10c1a1ca4d9f4b2bd5b89f7bbcbbc2d69e166fe23662b8db4f6beae0f50ac9fd",
+      "0xaa58a6545677c796a56b8bc874174c8cfd31a6c6e6ca3a87e086d4f66d52858a"
+    ],
+    transactions_root: "0xde8d25c0b9b54310128a21601331094b43f910f9f96102869c2e2dca94884bf4",
+    uncles: []
+  }
+
+  @blocks [@block, @block]
+
+  describe "transform_blocks/1" do
+    setup do
+      original = Application.get_env(:indexer, :block_transformer)
+
+      on_exit(fn -> Application.put_env(:indexer, :block_transformer, original) end)
+    end
+
+    test "transforms a list of blocks" do
+      assert Transform.transform_blocks(@blocks)
+    end
+
+    test "raises when no transformer is configured" do
+      Application.put_env(:indexer, :block_transformer, nil)
+
+      assert_raise ArgumentError, fn -> Transform.transform_blocks(@blocks) end
+    end
+  end
+end

apps/indexer/test/indexer/block/uncle/fetcher_test.exs

@@ -69,6 +69,9 @@ defmodule Indexer.Block.Uncle.FetcherTest do
                "number" => number_quantity,
                "parentHash" => "0x006edcaa1e6fde822908783bc4ef1ad3675532d542fce53537557391cfe34c3c",
                "size" => "0x243",
+               "receiptsRoot" => "0x0",
+               "sha3Uncles" => "0x0",
+               "stateRoot" => "0x0",
                "timestamp" => "0x5b437f41",
                "totalDifficulty" => "0x342337ffffffffffffffffffffffffed8d29bb",
                "transactions" => [
@@ -93,6 +96,7 @@ defmodule Indexer.Block.Uncle.FetcherTest do
                    "value" => "0x0"
                  }
                ],
+               "transactionsRoot" => "0x0",
                "uncles" => [uncle_uncle_hash_data]
              }
            }

apps/indexer/test/indexer/block/util_test.exs

@@ -0,0 +1,40 @@
+defmodule Indexer.Block.UtilTest do
+  use ExUnit.Case
+
+  alias Indexer.Block.Util
+
+  test "signer/1" do
+    data = %{
+      difficulty: 1,
+      extra_data:
+        "0xd68301080d846765746886676f312e3130856c696e7578000000000000000000773ab2ca8f47904a14739ad80a75b71d9d29b9fff8b7ecdcb73efffa6f74122f17d304b5dc8e6e5f256c9474dd115c8d4dae31b7a3d409e5c3270f8fde41cd8c00",
+      gas_limit: 7_753_377,
+      gas_used: 1_810_195,
+      hash: "0x7004c895e812c55b0c2be8a46d72ca300a683dc27d1d7917ee7742d4d0359c1f",
+      logs_bloom:
+        "0x00000000000000020000000000002000000400000000000000000000000000000000000000000000040000080004000020000010000000000000000000000000000000000000000008000008000000000000000000200000000000000000000000000000020000000000000000000800000000000000804000000010080000000800000000000000000000000000000000000000000000800000000000080000000008000400000000404000000000000000000000000200000000000000000000000002000000000000001002000000000000002000000008000000000020000000000000000000000000000000000000000000000000400000800000000000",
+      miner_hash: "0x0000000000000000000000000000000000000000",
+      mix_hash: "0x0000000000000000000000000000000000000000000000000000000000000000",
+      nonce: "0x0000000000000000",
+      number: 2_848_394,
+      parent_hash: "0x20350fc367e19d3865be1ea7da72ab81f8f9941c43ac6bb24a34a0a7caa2f3df",
+      receipts_root: "0x6ade4ac1079ea50cfadcce2b75ffbe4f9b14bf69b4607bbf1739463076ca6246",
+      sha3_uncles: "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347",
+      size: 6437,
+      state_root: "0x23f63347851bcd109059d007d71e19c4f5e73b7f0862bebcd04458333a004d92",
+      timestamp: DateTime.from_unix!(1_534_796_040),
+      total_difficulty: 5_353_647,
+      transactions: [
+        "0x7e3bb851fc74a436826d2af6b96e4db9484431811ef0d9c9e78370488d33d4e5",
+        "0x3976fd1e3d2a715c3cfcfde9bd3210798c26c017b8edb841d319227ecb3322fb",
+        "0xd8db124005bb8b6fda7b71fd56ac782552a66af58fe843ba3c4930423b87d1d2",
+        "0x10c1a1ca4d9f4b2bd5b89f7bbcbbc2d69e166fe23662b8db4f6beae0f50ac9fd",
+        "0xaa58a6545677c796a56b8bc874174c8cfd31a6c6e6ca3a87e086d4f66d52858a"
+      ],
+      transactions_root: "0xde8d25c0b9b54310128a21601331094b43f910f9f96102869c2e2dca94884bf4",
+      uncles: []
+    }
+
+    assert Util.signer(data) == "0xfc18cbc391de84dbd87db83b20935d3e89f5dd91"
+  end
+end

mix.lock

@@ -32,6 +32,7 @@
   "ex_cldr_units": {:hex, :ex_cldr_units, "1.1.1", "b3c7256709bdeb3740a5f64ce2bce659eb9cf4cc1afb4cf94aba033b4a18bc5f", [:mix], [{:ex_cldr, "~> 1.0", [hex: :ex_cldr, optional: false]}, {:ex_cldr_numbers, "~> 1.0", [hex: :ex_cldr_numbers, optional: false]}]},
   "ex_doc": {:hex, :ex_doc, "0.19.1", "519bb9c19526ca51d326c060cb1778d4a9056b190086a8c6c115828eaccea6cf", [:mix], [{:earmark, "~> 1.1", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.7", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm"},
   "ex_machina": {:hex, :ex_machina, "2.2.1", "df84d0b23487aaa8570c35e586d7f9f197a7787e1121344a41d8832a7ea41edf", [:mix], [{:ecto, "~> 2.1", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"},
+  "ex_rlp": {:hex, :ex_rlp, "0.3.1", "190554f7b26f79734fc5a772241eec14a71b2e83576e43f451479feb017013e9", [:mix], [], "hexpm"},
   "exactor": {:hex, :exactor, "2.2.4", "5efb4ddeb2c48d9a1d7c9b465a6fffdd82300eb9618ece5d34c3334d5d7245b1", [:mix], []},
   "excoveralls": {:git, "https://github.com/KronicDeth/excoveralls.git", "0a859b68851eeba9b43eba59fbc8f9098299cfe1", [branch: "circle-workflows"]},
   "exjsx": {:hex, :exjsx, "4.0.0", "60548841e0212df401e38e63c0078ec57b33e7ea49b032c796ccad8cde794b5c", [:mix], [{:jsx, "~> 2.8.0", [hex: :jsx, optional: false]}]},
@@ -50,7 +51,7 @@
   "jsx": {:hex, :jsx, "2.8.3", "a05252d381885240744d955fbe3cf810504eb2567164824e19303ea59eef62cf", [:mix, :rebar3], []},
   "junit_formatter": {:hex, :junit_formatter, "2.2.0", "da6093f0740c58a824f9585ebb7cb1b960efaecf48d1fa969e95d9c47c6b19dd", [:mix], [], "hexpm"},
   "keccakf1600": {:hex, :keccakf1600_orig, "2.0.0", "0a7217ddb3ee8220d449bbf7575ec39d4e967099f220a91e3dfca4dbaef91963", [:rebar3], []},
-  "libsecp256k1": {:hex, :libsecp256k1, "0.1.4", "42b7f76d8e32f85f578ccda0abfdb1afa0c5c231d1fd8aeab9cda352731a2d83", [:rebar3], []},
+  "libsecp256k1": {:hex, :libsecp256k1, "0.1.10", "d27495e2b9851c7765129b76c53b60f5e275bd6ff68292c50536bf6b8d091a4d", [:make, :mix], [{:mix_erlang_tasks, "0.1.0", [hex: :mix_erlang_tasks, repo: "hexpm", optional: false]}], "hexpm"},
   "logger_file_backend": {:hex, :logger_file_backend, "0.0.10", "876f9f84ae110781207c54321ffbb62bebe02946fe3c13f0d7c5f5d8ad4fa910", [:mix], [], "hexpm"},
   "makeup": {:hex, :makeup, "0.5.5", "9e08dfc45280c5684d771ad58159f718a7b5788596099bdfb0284597d368a882", [:mix], [{:nimble_parsec, "~> 0.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm"},
   "makeup_elixir": {:hex, :makeup_elixir, "0.10.0", "0f09c2ddf352887a956d84f8f7e702111122ca32fbbc84c2f0569b8b65cbf7fa", [:mix], [{:makeup, "~> 0.5.5", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm"},
@@ -59,6 +60,7 @@
   "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], []},
   "mime": {:hex, :mime, "1.3.0", "5e8d45a39e95c650900d03f897fbf99ae04f60ab1daa4a34c7a20a5151b7a5fe", [:mix], [], "hexpm"},
   "mimerl": {:hex, :mimerl, "1.0.2", "993f9b0e084083405ed8252b99460c4f0563e41729ab42d9074fd5e52439be88", [:rebar3], []},
+  "mix_erlang_tasks": {:hex, :mix_erlang_tasks, "0.1.0", "36819fec60b80689eb1380938675af215565a89320a9e29c72c70d97512e4649", [:mix], [], "hexpm"},
   "mochiweb": {:hex, :mochiweb, "2.18.0", "eb55f1db3e6e960fac4e6db4e2db9ec3602cc9f30b86cd1481d56545c3145d2e", [:rebar3], [], "hexpm"},
   "mock": {:hex, :mock, "0.3.2", "e98e998fd76c191c7e1a9557c8617912c53df3d4a6132f561eb762b699ef59fa", [:mix], [{:meck, "~> 0.8.8", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm"},
   "mox": {:hex, :mox, "0.4.0", "7f120840f7d626184a3d65de36189ca6f37d432e5d63acd80045198e4c5f7e6e", [:mix], [], "hexpm"},