Create task to fetch a single block and record it

README.md

@@ -22,7 +22,7 @@ This is a tool for inspecting and analyzing the POA Network blockchain.
 
 To get POA Explorer up and running locally:
 
-  * Set up some default configuration with: `$ cp config/dev.secrets.esx.example config/dev.secrets.esx`
+  * Set up some default configuration with: `$ cp config/dev.secret.esx.example config/dev.secret.esx`
   * Install dependencies with `$ mix do deps.get, local.rebar, deps.compile, compile`
   * Create and migrate your database with `$ mix ecto.create && mix ecto.migrate`
   * Install Node.js dependencies with `$ cd assets && npm install && cd ..`
@@ -38,10 +38,9 @@ Configure your local CCMenu with the following url: [`https://circleci.com/gh/po
 
 ### Testing
 
-To run the test suite: `$ mix test`
+  * Run the test suite: `$ mix test`
-
+  * Ensure your Elixir code is properly formatted: `$ mix credo --strict`
-To ensure your Elixir code is properly formatted: `$ mix credo --strict`
+  * Ensure your ES code is properly formatted: `$ cd assets && npm run eslint`
-To ensure your ES code is properly formatted: `$ cd assets && npm run eslint`
 
 
 ## Internationalization

lib/explorer/fetcher.ex

@@ -0,0 +1,49 @@
+defmodule Explorer.Fetcher  do
+  alias Explorer.Block
+  alias Explorer.Repo
+  import Ethereumex.HttpClient, only: [eth_get_block_by_number: 2]
+
+  @moduledoc false
+
+  def fetch(block) do
+    block
+    |> download_block
+    |> extract_block
+    |> validate_block
+    |> Repo.insert
+  end
+
+  def download_block(block_number) do
+    {:ok, block} = eth_get_block_by_number(block_number, true)
+    block
+  end
+
+  def extract_block(block) do
+    %Block{
+      hash: block["hash"],
+      number: block["number"] |> decode_integer_field,
+      gas_used: block["gasUsed"] |> decode_integer_field,
+      timestamp: block["timestamp"] |> decode_time_field,
+      parent_hash: block["parentHash"],
+      miner: block["miner"],
+      difficulty: block["difficulty"] |> decode_integer_field,
+      total_difficulty: block["totalDifficulty"] |> decode_integer_field,
+      size: block["size"] |> decode_integer_field,
+      gas_limit: block["gasLimit"] |> decode_integer_field,
+      nonce: block["nonce"] || "0",
+    }
+  end
+
+  def validate_block(struct) do
+    Block.changeset(struct, %{})
+  end
+
+  def decode_integer_field(hex) do
+    {"0x", base_16} = String.split_at(hex, 2)
+    String.to_integer(base_16, 16)
+  end
+
+  def decode_time_field(field) do
+    field |> decode_integer_field |> Timex.from_unix
+  end
+end

lib/explorer/latest_block.ex

@@ -0,0 +1,15 @@
+defmodule Explorer.LatestBlock do
+  alias Explorer.Fetcher
+  import Ethereumex.HttpClient, only: [eth_block_number: 0]
+
+  @moduledoc false
+
+  def fetch do
+    get_latest_block() |> Fetcher.fetch
+  end
+
+  def get_latest_block do
+    {:ok, block_number} = eth_block_number()
+    block_number
+  end
+end

lib/mix/tasks/scrape.ex

@@ -1,72 +1,12 @@
 defmodule Mix.Tasks.Scrape do
   use Mix.Task
-  alias Explorer.Block
+  alias Explorer.LatestBlock
-  alias Explorer.Repo
-  import Ethereumex.HttpClient, only: [
-    eth_block_number: 0,
-    eth_get_block_by_number: 2
-  ]
 
   @shortdoc "Scrape the blockchain."
   @moduledoc false
 
   def run(_) do
-    persist()
-  end
-
-  def persist do
-    {:ok, _} = Repo.insert(changeset())
-  end
-
-  def changeset do
-    Block.changeset(attributes(), %{})
-  end
-
-  def attributes do
-    %Block{
-      hash: hash(),
-      number: number(),
-      timestamp: timestamp(),
-      gas_used: gas_used(),
-      parent_hash: "0x0",
-      nonce: "0",
-      miner: "0x0",
-      difficulty: 0,
-      total_difficulty: 0,
-      size: 0,
-      gas_limit: 0,
-    }
-  end
-
-  def hash do
-    latest_block()["hash"]
-  end
-
-  def number do
-    decode_integer_field(latest_block()["number"])
-  end
-
-  def timestamp do
-    epoch_to_datetime(decode_integer_field(latest_block()["timestamp"]))
-  end
-
-  def gas_used do
-    decode_integer_field(latest_block()["gasUsed"])
-  end
-
-  def latest_block do
     Mix.Task.run "app.start"
-    {:ok, latest_block_number} = eth_block_number()
+    LatestBlock.fetch()
-    {:ok, latest_block} = eth_get_block_by_number(latest_block_number, true)
-    latest_block
-  end
-
-  def decode_integer_field(field) do
-    {"0x", base_16} = String.split_at(field, 2)
-    String.to_integer(base_16, 16)
-  end
-
-  def epoch_to_datetime(epoch) do
-    Timex.from_unix(epoch)
   end
 end

mix.exs

@@ -43,6 +43,7 @@ defmodule Explorer.Mixfile do
       {:credo, "~> 0.8", only: [:dev, :test], runtime: false},
       {:ethereumex, github: "exthereum/ethereumex", commit: "262f1d81ae163ffb46e127283658249dac1c8318"}, # Waiting for this version to be pushed to Hex.
       {:ex_machina, "~> 2.1", only: [:test]},
+      {:exvcr, "~> 0.8", only: :test},
       {:gettext, "~> 0.11"},
       {:junit_formatter, ">= 0.0.0"},
       {:new_relixir, "~> 0.4.0", only: [:prod]},

mix.lock

@@ -10,12 +10,17 @@
   "ecto": {:hex, :ecto, "2.2.8", "a4463c0928b970f2cee722cd29aaac154e866a15882c5737e0038bbfcf03ec2c", [:mix], [{:db_connection, "~> 1.1", [hex: :db_connection, repo: "hexpm", optional: true]}, {:decimal, "~> 1.2", [hex: :decimal, repo: "hexpm", optional: false]}, {:mariaex, "~> 0.8.0", [hex: :mariaex, repo: "hexpm", optional: true]}, {:poison, "~> 2.2 or ~> 3.0", [hex: :poison, repo: "hexpm", optional: true]}, {:poolboy, "~> 1.5", [hex: :poolboy, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.13.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:sbroker, "~> 1.0", [hex: :sbroker, repo: "hexpm", optional: true]}], "hexpm"},
   "ethereumex": {:git, "https://github.com/exthereum/ethereumex.git", "262f1d81ae163ffb46e127283658249dac1c8318", []},
   "ex_machina": {:hex, :ex_machina, "2.1.0", "4874dc9c78e7cf2d429f24dc3c4005674d4e4da6a08be961ffccc08fb528e28b", [:mix], [{:ecto, "~> 2.1", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"},
+  "exactor": {:hex, :exactor, "2.2.4", "5efb4ddeb2c48d9a1d7c9b465a6fffdd82300eb9618ece5d34c3334d5d7245b1", [:mix], [], "hexpm"},
+  "exjsx": {:hex, :exjsx, "4.0.0", "60548841e0212df401e38e63c0078ec57b33e7ea49b032c796ccad8cde794b5c", [:mix], [{:jsx, "~> 2.8.0", [hex: :jsx, repo: "hexpm", optional: false]}], "hexpm"},
+  "exvcr": {:hex, :exvcr, "0.9.1", "31e3936a790a14bf56b31b6b276577076a5ef8afd9b2d53ba3ff8bb647d45613", [:mix], [{:exactor, "~> 2.2", [hex: :exactor, repo: "hexpm", optional: false]}, {:exjsx, "~> 4.0", [hex: :exjsx, repo: "hexpm", optional: false]}, {:httpoison, "~> 0.13", [hex: :httpoison, repo: "hexpm", optional: true]}, {:httpotion, "~> 3.0", [hex: :httpotion, repo: "hexpm", optional: true]}, {:ibrowse, "~> 4.4", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:meck, "~> 0.8.8", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm"},
   "file_system": {:hex, :file_system, "0.2.4", "f0bdda195c0e46e987333e986452ec523aed21d784189144f647c43eaf307064", [:mix], [], "hexpm"},
   "gettext": {:hex, :gettext, "0.14.0", "1a019a2e51d5ad3d126efe166dcdf6563768e5d06c32a99ad2281a1fa94b4c72", [:mix], [], "hexpm"},
   "hackney": {:hex, :hackney, "1.10.1", "c38d0ca52ea80254936a32c45bb7eb414e7a96a521b4ce76d00a69753b157f21", [:rebar3], [{:certifi, "2.0.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "5.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "1.0.2", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"},
   "httpoison": {:hex, :httpoison, "0.13.0", "bfaf44d9f133a6599886720f3937a7699466d23bb0cd7a88b6ba011f53c6f562", [:mix], [{:hackney, "~> 1.8", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"},
   "idna": {:hex, :idna, "5.1.0", "d72b4effeb324ad5da3cab1767cb16b17939004e789d8c0ad5b70f3cea20c89a", [:rebar3], [{:unicode_util_compat, "0.3.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm"},
+  "jsx": {:hex, :jsx, "2.8.3", "a05252d381885240744d955fbe3cf810504eb2567164824e19303ea59eef62cf", [:mix, :rebar3], [], "hexpm"},
   "junit_formatter": {:hex, :junit_formatter, "2.1.0", "ff03d2bbe9a67041f2488d8e72180ddcf4dff9e9c8a39b79eac9828fcb9e9bbf", [:mix], [], "hexpm"},
+  "meck": {:hex, :meck, "0.8.9", "64c5c0bd8bcca3a180b44196265c8ed7594e16bcc845d0698ec6b4e577f48188", [:rebar3], [], "hexpm"},
   "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm"},
   "mime": {:hex, :mime, "1.2.0", "78adaa84832b3680de06f88f0997e3ead3b451a440d183d688085be2d709b534", [:mix], [], "hexpm"},
   "mimerl": {:hex, :mimerl, "1.0.2", "993f9b0e084083405ed8252b99460c4f0563e41729ab42d9074fd5e52439be88", [:rebar3], [], "hexpm"},

test/explorer/block_test.exs

@@ -4,6 +4,11 @@ defmodule Explorer.BlockTest do
   alias Explorer.Block
 
   describe "changeset/2" do
+    test "with valid attributes" do
+      changeset = build(:block) |> Block.changeset(%{})
+      assert(changeset.valid?)
+    end
+
     test "with invalid attributes" do
       changeset = %Block{} |> Block.changeset(%{racecar: "yellow ham"})
       refute(changeset.valid?)

test/explorer/fetcher_test.exs

@@ -0,0 +1,91 @@
+defmodule Explorer.FetcherTest do
+  use Explorer.DataCase
+
+  alias Explorer.Block
+  alias Explorer.Repo
+  alias Explorer.Fetcher
+
+  describe "fetch/1" do
+    test "the latest block is copied over from the blockchain" do
+      use_cassette "fetcher_fetch" do
+        Fetcher.fetch("0x89923")
+
+        last_block = Block
+          |> order_by(desc: :inserted_at)
+          |> limit(1)
+          |> Repo.all
+          |> List.first
+
+        assert last_block.number == 563491
+      end
+    end
+  end
+
+  describe "download_block/1" do
+    test "returns a block for a given block number" do
+      use_cassette "fetcher_download_block" do
+        assert Fetcher.download_block("0x89923")["hash"] == "0x342878f5a2c06bc6146f9440910bab9c5ddae5dbd13c9a01d8adaf51ff5593ae"
+      end
+    end
+  end
+
+  describe "extract_block/1" do
+    def raw_block(nonce \\ %{}) do
+      Map.merge(%{
+        "difficulty" => "0xfffffffffffffffffffffffffffffffe",
+        "gasLimit" => "0x02",
+        "gasUsed" => "0x19522",
+        "hash" => "bananas",
+        "miner" => "0xdb1207770e0a4258d7a4ce49ab037f92564fea85",
+        "number" => "0x7f2fb",
+        "parentHash" => "0x70029f66ea5a3b2b1ede95079d95a2ab74b649b5b17cdcf6f29b6317e7c7efa6",
+        "size" => "0x10",
+        "timestamp" => "0x12",
+        "totalDifficulty" => "0xff",
+      }, nonce)
+    end
+
+    test "returns the struct of a block" do
+      processed_block = %Block{
+        difficulty: 340282366920938463463374607431768211454,
+        gas_limit: 2,
+        gas_used: 103714,
+        hash: "bananas",
+        nonce: "0xfb6e1a62d119228b",
+        miner: "0xdb1207770e0a4258d7a4ce49ab037f92564fea85",
+        number: 520955,
+        parent_hash: "0x70029f66ea5a3b2b1ede95079d95a2ab74b649b5b17cdcf6f29b6317e7c7efa6",
+        size: 16,
+        timestamp: Timex.parse!("1970-01-01T00:00:18-00:00", "{ISO:Extended}"),
+        total_difficulty: 255,
+      }
+      assert Fetcher.extract_block(raw_block(%{"nonce" => "0xfb6e1a62d119228b"})) == processed_block
+    end
+
+    test "when there is no nonce" do
+      assert Fetcher.extract_block(raw_block()).nonce == "0"
+    end
+  end
+
+  describe "decode_integer_field/1" do
+    test "returns the integer value of a hex value" do
+      assert(Fetcher.decode_integer_field("0x7f2fb") == 520955)
+    end
+  end
+
+  describe "decode_time_field/1" do
+    test "returns the date value of a hex value" do
+      the_seventies = Timex.parse!("1970-01-01T00:00:18-00:00", "{ISO:Extended}")
+      assert(Fetcher.decode_time_field("0x12") == the_seventies)
+    end
+  end
+
+  describe "validate_block/1" do
+    test "returns a valid changeset for an extracted block" do
+      use_cassette "fetcher_validate_block" do
+        changeset = Fetcher.download_block("0x89923") |> Fetcher.extract_block |> Fetcher.validate_block
+        assert(changeset.valid?)
+      end
+    end
+  end
+end

test/explorer/latest_block_test.exs

@@ -0,0 +1,31 @@
+defmodule Explorer.LatestBlockTest do
+  use Explorer.DataCase
+
+  alias Explorer.Block
+  alias Explorer.Repo
+  alias Explorer.LatestBlock
+
+  describe "fetch/0" do
+    test "the latest block is copied over from the blockchain" do
+      use_cassette "latest_block_fetch" do
+        LatestBlock.fetch()
+
+        last_block = Block
+          |> order_by(desc: :inserted_at)
+          |> limit(1)
+          |> Repo.all
+          |> List.first
+
+        assert(last_block.number)
+      end
+    end
+  end
+
+  describe "get_latest_block/0" do
+    test "returns the number of the latest block" do
+      use_cassette "fetcher_get_latest_block" do
+        assert LatestBlock.get_latest_block() == "0x89923"
+      end
+    end
+  end
+end

test/mix/tasks/scrape_test.exs

@@ -4,14 +4,16 @@ defmodule Scrape.Test do
   alias Explorer.Repo
 
   test "it downloads a new block" do
-    Mix.Tasks.Scrape.run([])
+    use_cassette "scrape" do
+      Mix.Tasks.Scrape.run([])
 
-    last_block = Block
+      last_block = Block
-      |> order_by(desc: :inserted_at)
+        |> order_by(desc: :inserted_at)
-      |> limit(1)
+        |> limit(1)
-      |> Repo.all
+        |> Repo.all
-      |> List.first
+        |> List.first
 
-    assert(last_block.number)
+      assert(last_block.number)
+    end
   end
 end

test/support/data_case.ex

@@ -18,6 +18,8 @@ defmodule Explorer.DataCase do
     quote do
       alias Explorer.Repo
 
+      use ExVCR.Mock, adapter: ExVCR.Adapter.Hackney
+
       import Ecto
       import Ecto.Changeset
       import Ecto.Query
@@ -27,6 +29,8 @@ defmodule Explorer.DataCase do
   end
 
   setup tags do
+    ExVCR.Config.cassette_library_dir("test/support/fixture/vcr_cassettes")
+
     :ok = Ecto.Adapters.SQL.Sandbox.checkout(Explorer.Repo)
 
     unless tags[:async] do

test/support/fixture/vcr_cassettes/fetcher_download_block.json

@@ -0,0 +1,30 @@
+[
+  {
+    "request": {
+      "body": "{\"params\":[\"0x89923\",true],\"method\":\"eth_getBlockByNumber\",\"jsonrpc\":\"2.0\",\"id\":0}",
+      "headers": {
+        "Content-Type": "application/json"
+      },
+      "method": "post",
+      "options": [],
+      "request_body": "",
+      "url": "https://sokol.poa.network:443"
+    },
+    "response": {
+      "binary": false,
+      "body": "{\"jsonrpc\":\"2.0\",\"result\":{\"author\":\"0xdb1207770e0a4258d7a4ce49ab037f92564fea85\",\"difficulty\":\"0xfffffffffffffffffffffffffffffffe\",\"extraData\":\"0xd5830108048650617269747986312e32322e31826c69\",\"gasLimit\":\"0x7a1200\",\"gasUsed\":\"0x0\",\"hash\":\"0x342878f5a2c06bc6146f9440910bab9c5ddae5dbd13c9a01d8adaf51ff5593ae\",\"logsBloom\":\"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000\",\"miner\":\"0xdb1207770e0a4258d7a4ce49ab037f92564fea85\",\"number\":\"0x89923\",\"parentHash\":\"0x70029f66ea5a3b2b1ede95079d95a2ab74b649b5b17cdcf6f29b6317e7c7efa6\",\"receiptsRoot\":\"0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421\",\"sealFields\":[\"0x8412147192\",\"0xb841db669d5b8d22ef69c849a914682c242cde9492cbf1de381d0d4f1857d871963767470f38b1f63031fe47614175609989552ac5e8a6c2844f2d48dfdc48e9df4001\"],\"sha3Uncles\":\"0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347\",\"signature\":\"db669d5b8d22ef69c849a914682c242cde9492cbf1de381d0d4f1857d871963767470f38b1f63031fe47614175609989552ac5e8a6c2844f2d48dfdc48e9df4001\",\"size\":\"0x243\",\"stateRoot\":\"0x1ce8bea4dc0a87b99af15a65bb79980cb337e5bf4d1f5ad7d2d263572d177f1a\",\"step\":\"303329682\",\"timestamp\":\"0x5a6637da\",\"totalDifficulty\":\"0x89922ffffffffffffffffffffffffede4f54b\",\"transactions\":[],\"transactionsRoot\":\"0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421\",\"uncles\":[]},\"id\":0}\n",
+      "headers": {
+        "Date": "Tue, 23 Jan 2018 00:39:56 GMT",
+        "Content-Type": "application/json",
+        "Transfer-Encoding": "chunked",
+        "Connection": "keep-alive",
+        "Set-Cookie": "__cfduid=da35df504d925fe5d76380c5a13de56671516667996; expires=Wed, 23-Jan-19 00:39:56 GMT; path=/; domain=.poa.network; HttpOnly; Secure",
+        "Expect-CT": "max-age=604800, report-uri=\"https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct\"",
+        "Server": "cloudflare",
+        "CF-RAY": "3e16b2e0dd9c93ae-SJC"
+      },
+      "status_code": 200,
+      "type": "ok"
+    }
+  }
+]

test/support/fixture/vcr_cassettes/fetcher_fetch.json

@@ -0,0 +1,30 @@
+[
+  {
+    "request": {
+      "body": "{\"params\":[\"0x89923\",true],\"method\":\"eth_getBlockByNumber\",\"jsonrpc\":\"2.0\",\"id\":0}",
+      "headers": {
+        "Content-Type": "application/json"
+      },
+      "method": "post",
+      "options": [],
+      "request_body": "",
+      "url": "https://sokol.poa.network:443"
+    },
+    "response": {
+      "binary": false,
+      "body": "{\"jsonrpc\":\"2.0\",\"result\":{\"author\":\"0xdb1207770e0a4258d7a4ce49ab037f92564fea85\",\"difficulty\":\"0xfffffffffffffffffffffffffffffffe\",\"extraData\":\"0xd5830108048650617269747986312e32322e31826c69\",\"gasLimit\":\"0x7a1200\",\"gasUsed\":\"0x0\",\"hash\":\"0x342878f5a2c06bc6146f9440910bab9c5ddae5dbd13c9a01d8adaf51ff5593ae\",\"logsBloom\":\"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000\",\"miner\":\"0xdb1207770e0a4258d7a4ce49ab037f92564fea85\",\"number\":\"0x89923\",\"parentHash\":\"0x70029f66ea5a3b2b1ede95079d95a2ab74b649b5b17cdcf6f29b6317e7c7efa6\",\"receiptsRoot\":\"0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421\",\"sealFields\":[\"0x8412147192\",\"0xb841db669d5b8d22ef69c849a914682c242cde9492cbf1de381d0d4f1857d871963767470f38b1f63031fe47614175609989552ac5e8a6c2844f2d48dfdc48e9df4001\"],\"sha3Uncles\":\"0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347\",\"signature\":\"db669d5b8d22ef69c849a914682c242cde9492cbf1de381d0d4f1857d871963767470f38b1f63031fe47614175609989552ac5e8a6c2844f2d48dfdc48e9df4001\",\"size\":\"0x243\",\"stateRoot\":\"0x1ce8bea4dc0a87b99af15a65bb79980cb337e5bf4d1f5ad7d2d263572d177f1a\",\"step\":\"303329682\",\"timestamp\":\"0x5a6637da\",\"totalDifficulty\":\"0x89922ffffffffffffffffffffffffede4f54b\",\"transactions\":[],\"transactionsRoot\":\"0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421\",\"uncles\":[]},\"id\":0}\n",
+      "headers": {
+        "Date": "Mon, 22 Jan 2018 19:14:51 GMT",
+        "Content-Type": "application/json",
+        "Transfer-Encoding": "chunked",
+        "Connection": "keep-alive",
+        "Set-Cookie": "__cfduid=d6d7bf13bce28ec88dc130f482acb91ab1516648491; expires=Tue, 22-Jan-19 19:14:51 GMT; path=/; domain=.poa.network; HttpOnly; Secure",
+        "Expect-CT": "max-age=604800, report-uri=\"https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct\"",
+        "Server": "cloudflare",
+        "CF-RAY": "3e14d6ad693128be-SJC"
+      },
+      "status_code": 200,
+      "type": "ok"
+    }
+  }
+]

test/support/fixture/vcr_cassettes/fetcher_fetch_block.json

@@ -0,0 +1,30 @@
+[
+  {
+    "request": {
+      "body": "{\"params\":[\"0x89923\",true],\"method\":\"eth_getBlockByNumber\",\"jsonrpc\":\"2.0\",\"id\":0}",
+      "headers": {
+        "Content-Type": "application/json"
+      },
+      "method": "post",
+      "options": [],
+      "request_body": "",
+      "url": "https://sokol.poa.network:443"
+    },
+    "response": {
+      "binary": false,
+      "body": "{\"jsonrpc\":\"2.0\",\"result\":{\"author\":\"0xdb1207770e0a4258d7a4ce49ab037f92564fea85\",\"difficulty\":\"0xfffffffffffffffffffffffffffffffe\",\"extraData\":\"0xd5830108048650617269747986312e32322e31826c69\",\"gasLimit\":\"0x7a1200\",\"gasUsed\":\"0x0\",\"hash\":\"0x342878f5a2c06bc6146f9440910bab9c5ddae5dbd13c9a01d8adaf51ff5593ae\",\"logsBloom\":\"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000\",\"miner\":\"0xdb1207770e0a4258d7a4ce49ab037f92564fea85\",\"number\":\"0x89923\",\"parentHash\":\"0x70029f66ea5a3b2b1ede95079d95a2ab74b649b5b17cdcf6f29b6317e7c7efa6\",\"receiptsRoot\":\"0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421\",\"sealFields\":[\"0x8412147192\",\"0xb841db669d5b8d22ef69c849a914682c242cde9492cbf1de381d0d4f1857d871963767470f38b1f63031fe47614175609989552ac5e8a6c2844f2d48dfdc48e9df4001\"],\"sha3Uncles\":\"0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347\",\"signature\":\"db669d5b8d22ef69c849a914682c242cde9492cbf1de381d0d4f1857d871963767470f38b1f63031fe47614175609989552ac5e8a6c2844f2d48dfdc48e9df4001\",\"size\":\"0x243\",\"stateRoot\":\"0x1ce8bea4dc0a87b99af15a65bb79980cb337e5bf4d1f5ad7d2d263572d177f1a\",\"step\":\"303329682\",\"timestamp\":\"0x5a6637da\",\"totalDifficulty\":\"0x89922ffffffffffffffffffffffffede4f54b\",\"transactions\":[],\"transactionsRoot\":\"0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421\",\"uncles\":[]},\"id\":0}\n",
+      "headers": {
+        "Date": "Mon, 22 Jan 2018 19:14:51 GMT",
+        "Content-Type": "application/json",
+        "Transfer-Encoding": "chunked",
+        "Connection": "keep-alive",
+        "Set-Cookie": "__cfduid=d6d7bf13bce28ec88dc130f482acb91ab1516648491; expires=Tue, 22-Jan-19 19:14:51 GMT; path=/; domain=.poa.network; HttpOnly; Secure",
+        "Expect-CT": "max-age=604800, report-uri=\"https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct\"",
+        "Server": "cloudflare",
+        "CF-RAY": "3e14d6ad693128be-SJC"
+      },
+      "status_code": 200,
+      "type": "ok"
+    }
+  }
+]

test/support/fixture/vcr_cassettes/fetcher_get_latest_block.json

@@ -0,0 +1,30 @@
+[
+  {
+    "request": {
+      "body": "{\"params\":[],\"method\":\"eth_blockNumber\",\"jsonrpc\":\"2.0\",\"id\":0}",
+      "headers": {
+        "Content-Type": "application/json"
+      },
+      "method": "post",
+      "options": [],
+      "request_body": "",
+      "url": "https://sokol.poa.network:443"
+    },
+    "response": {
+      "binary": false,
+      "body": "{\"jsonrpc\":\"2.0\",\"result\":\"0x89923\",\"id\":0}\n",
+      "headers": {
+        "Date": "Mon, 22 Jan 2018 19:13:30 GMT",
+        "Content-Type": "application/json",
+        "Transfer-Encoding": "chunked",
+        "Connection": "keep-alive",
+        "Set-Cookie": "__cfduid=d5537ec24d731578ee1e6c5d48a0d252d1516648410; expires=Tue, 22-Jan-19 19:13:30 GMT; path=/; domain=.poa.network; HttpOnly; Secure",
+        "Expect-CT": "max-age=604800, report-uri=\"https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct\"",
+        "Server": "cloudflare",
+        "CF-RAY": "3e14d4b5ce139607-SJC"
+      },
+      "status_code": 200,
+      "type": "ok"
+    }
+  }
+]

test/support/fixture/vcr_cassettes/fetcher_validate_block.json

@@ -0,0 +1,30 @@
+[
+  {
+    "request": {
+      "body": "{\"params\":[\"0x89923\",true],\"method\":\"eth_getBlockByNumber\",\"jsonrpc\":\"2.0\",\"id\":0}",
+      "headers": {
+        "Content-Type": "application/json"
+      },
+      "method": "post",
+      "options": [],
+      "request_body": "",
+      "url": "https://sokol.poa.network:443"
+    },
+    "response": {
+      "binary": false,
+      "body": "{\"jsonrpc\":\"2.0\",\"result\":{\"author\":\"0xdb1207770e0a4258d7a4ce49ab037f92564fea85\",\"difficulty\":\"0xfffffffffffffffffffffffffffffffe\",\"extraData\":\"0xd5830108048650617269747986312e32322e31826c69\",\"gasLimit\":\"0x7a1200\",\"gasUsed\":\"0x0\",\"hash\":\"0x342878f5a2c06bc6146f9440910bab9c5ddae5dbd13c9a01d8adaf51ff5593ae\",\"logsBloom\":\"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000\",\"miner\":\"0xdb1207770e0a4258d7a4ce49ab037f92564fea85\",\"number\":\"0x89923\",\"parentHash\":\"0x70029f66ea5a3b2b1ede95079d95a2ab74b649b5b17cdcf6f29b6317e7c7efa6\",\"receiptsRoot\":\"0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421\",\"sealFields\":[\"0x8412147192\",\"0xb841db669d5b8d22ef69c849a914682c242cde9492cbf1de381d0d4f1857d871963767470f38b1f63031fe47614175609989552ac5e8a6c2844f2d48dfdc48e9df4001\"],\"sha3Uncles\":\"0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347\",\"signature\":\"db669d5b8d22ef69c849a914682c242cde9492cbf1de381d0d4f1857d871963767470f38b1f63031fe47614175609989552ac5e8a6c2844f2d48dfdc48e9df4001\",\"size\":\"0x243\",\"stateRoot\":\"0x1ce8bea4dc0a87b99af15a65bb79980cb337e5bf4d1f5ad7d2d263572d177f1a\",\"step\":\"303329682\",\"timestamp\":\"0x5a6637da\",\"totalDifficulty\":\"0x89922ffffffffffffffffffffffffede4f54b\",\"transactions\":[],\"transactionsRoot\":\"0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421\",\"uncles\":[]},\"id\":0}\n",
+      "headers": {
+        "Date": "Mon, 22 Jan 2018 19:14:51 GMT",
+        "Content-Type": "application/json",
+        "Transfer-Encoding": "chunked",
+        "Connection": "keep-alive",
+        "Set-Cookie": "__cfduid=d6d7bf13bce28ec88dc130f482acb91ab1516648491; expires=Tue, 22-Jan-19 19:14:51 GMT; path=/; domain=.poa.network; HttpOnly; Secure",
+        "Expect-CT": "max-age=604800, report-uri=\"https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct\"",
+        "Server": "cloudflare",
+        "CF-RAY": "3e14d6ad693128be-SJC"
+      },
+      "status_code": 200,
+      "type": "ok"
+    }
+  }
+]

test/support/fixture/vcr_cassettes/latest_block_fetch.json

@@ -0,0 +1,58 @@
+[
+  {
+    "request": {
+      "body": "{\"params\":[],\"method\":\"eth_blockNumber\",\"jsonrpc\":\"2.0\",\"id\":0}",
+      "headers": {
+        "Content-Type": "application/json"
+      },
+      "method": "post",
+      "options": [],
+      "request_body": "",
+      "url": "https://sokol.poa.network:443"
+    },
+    "response": {
+      "binary": false,
+      "body": "{\"jsonrpc\":\"2.0\",\"result\":\"0x89923\",\"id\":0}\n",
+      "headers": {
+        "Date": "Mon, 22 Jan 2018 19:13:30 GMT",
+        "Content-Type": "application/json",
+        "Transfer-Encoding": "chunked",
+        "Connection": "keep-alive",
+        "Set-Cookie": "__cfduid=d5537ec24d731578ee1e6c5d48a0d252d1516648410; expires=Tue, 22-Jan-19 19:13:30 GMT; path=/; domain=.poa.network; HttpOnly; Secure",
+        "Expect-CT": "max-age=604800, report-uri=\"https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct\"",
+        "Server": "cloudflare",
+        "CF-RAY": "3e14d4b5ce139607-SJC"
+      },
+      "status_code": 200,
+      "type": "ok"
+    }
+  },
+  {
+    "request": {
+      "body": "{\"params\":[\"0x89923\",true],\"method\":\"eth_getBlockByNumber\",\"jsonrpc\":\"2.0\",\"id\":0}",
+      "headers": {
+        "Content-Type": "application/json"
+      },
+      "method": "post",
+      "options": [],
+      "request_body": "",
+      "url": "https://sokol.poa.network:443"
+    },
+    "response": {
+      "binary": false,
+      "body": "{\"jsonrpc\":\"2.0\",\"result\":{\"author\":\"0xdb1207770e0a4258d7a4ce49ab037f92564fea85\",\"difficulty\":\"0xfffffffffffffffffffffffffffffffe\",\"extraData\":\"0xd5830108048650617269747986312e32322e31826c69\",\"gasLimit\":\"0x7a1200\",\"gasUsed\":\"0x0\",\"hash\":\"0x342878f5a2c06bc6146f9440910bab9c5ddae5dbd13c9a01d8adaf51ff5593ae\",\"logsBloom\":\"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000\",\"miner\":\"0xdb1207770e0a4258d7a4ce49ab037f92564fea85\",\"number\":\"0x89923\",\"parentHash\":\"0x70029f66ea5a3b2b1ede95079d95a2ab74b649b5b17cdcf6f29b6317e7c7efa6\",\"receiptsRoot\":\"0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421\",\"sealFields\":[\"0x8412147192\",\"0xb841db669d5b8d22ef69c849a914682c242cde9492cbf1de381d0d4f1857d871963767470f38b1f63031fe47614175609989552ac5e8a6c2844f2d48dfdc48e9df4001\"],\"sha3Uncles\":\"0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347\",\"signature\":\"db669d5b8d22ef69c849a914682c242cde9492cbf1de381d0d4f1857d871963767470f38b1f63031fe47614175609989552ac5e8a6c2844f2d48dfdc48e9df4001\",\"size\":\"0x243\",\"stateRoot\":\"0x1ce8bea4dc0a87b99af15a65bb79980cb337e5bf4d1f5ad7d2d263572d177f1a\",\"step\":\"303329682\",\"timestamp\":\"0x5a6637da\",\"totalDifficulty\":\"0x89922ffffffffffffffffffffffffede4f54b\",\"transactions\":[],\"transactionsRoot\":\"0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421\",\"uncles\":[]},\"id\":0}\n",
+      "headers": {
+        "Date": "Mon, 22 Jan 2018 19:14:51 GMT",
+        "Content-Type": "application/json",
+        "Transfer-Encoding": "chunked",
+        "Connection": "keep-alive",
+        "Set-Cookie": "__cfduid=d6d7bf13bce28ec88dc130f482acb91ab1516648491; expires=Tue, 22-Jan-19 19:14:51 GMT; path=/; domain=.poa.network; HttpOnly; Secure",
+        "Expect-CT": "max-age=604800, report-uri=\"https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct\"",
+        "Server": "cloudflare",
+        "CF-RAY": "3e14d6ad693128be-SJC"
+      },
+      "status_code": 200,
+      "type": "ok"
+    }
+  }
+]

test/support/fixture/vcr_cassettes/scrape.json

@@ -0,0 +1,58 @@
+[
+  {
+    "request": {
+      "body": "{\"params\":[],\"method\":\"eth_blockNumber\",\"jsonrpc\":\"2.0\",\"id\":0}",
+      "headers": {
+        "Content-Type": "application/json"
+      },
+      "method": "post",
+      "options": [],
+      "request_body": "",
+      "url": "https://sokol.poa.network:443"
+    },
+    "response": {
+      "binary": false,
+      "body": "{\"jsonrpc\":\"2.0\",\"result\":\"0x89923\",\"id\":0}\n",
+      "headers": {
+        "Date": "Mon, 22 Jan 2018 19:13:30 GMT",
+        "Content-Type": "application/json",
+        "Transfer-Encoding": "chunked",
+        "Connection": "keep-alive",
+        "Set-Cookie": "__cfduid=d5537ec24d731578ee1e6c5d48a0d252d1516648410; expires=Tue, 22-Jan-19 19:13:30 GMT; path=/; domain=.poa.network; HttpOnly; Secure",
+        "Expect-CT": "max-age=604800, report-uri=\"https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct\"",
+        "Server": "cloudflare",
+        "CF-RAY": "3e14d4b5ce139607-SJC"
+      },
+      "status_code": 200,
+      "type": "ok"
+    }
+  },
+  {
+    "request": {
+      "body": "{\"params\":[\"0x89923\",true],\"method\":\"eth_getBlockByNumber\",\"jsonrpc\":\"2.0\",\"id\":0}",
+      "headers": {
+        "Content-Type": "application/json"
+      },
+      "method": "post",
+      "options": [],
+      "request_body": "",
+      "url": "https://sokol.poa.network:443"
+    },
+    "response": {
+      "binary": false,
+      "body": "{\"jsonrpc\":\"2.0\",\"result\":{\"author\":\"0xdb1207770e0a4258d7a4ce49ab037f92564fea85\",\"difficulty\":\"0xfffffffffffffffffffffffffffffffe\",\"extraData\":\"0xd5830108048650617269747986312e32322e31826c69\",\"gasLimit\":\"0x7a1200\",\"gasUsed\":\"0x0\",\"hash\":\"0x342878f5a2c06bc6146f9440910bab9c5ddae5dbd13c9a01d8adaf51ff5593ae\",\"logsBloom\":\"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000\",\"miner\":\"0xdb1207770e0a4258d7a4ce49ab037f92564fea85\",\"number\":\"0x89923\",\"parentHash\":\"0x70029f66ea5a3b2b1ede95079d95a2ab74b649b5b17cdcf6f29b6317e7c7efa6\",\"receiptsRoot\":\"0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421\",\"sealFields\":[\"0x8412147192\",\"0xb841db669d5b8d22ef69c849a914682c242cde9492cbf1de381d0d4f1857d871963767470f38b1f63031fe47614175609989552ac5e8a6c2844f2d48dfdc48e9df4001\"],\"sha3Uncles\":\"0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347\",\"signature\":\"db669d5b8d22ef69c849a914682c242cde9492cbf1de381d0d4f1857d871963767470f38b1f63031fe47614175609989552ac5e8a6c2844f2d48dfdc48e9df4001\",\"size\":\"0x243\",\"stateRoot\":\"0x1ce8bea4dc0a87b99af15a65bb79980cb337e5bf4d1f5ad7d2d263572d177f1a\",\"step\":\"303329682\",\"timestamp\":\"0x5a6637da\",\"totalDifficulty\":\"0x89922ffffffffffffffffffffffffede4f54b\",\"transactions\":[],\"transactionsRoot\":\"0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421\",\"uncles\":[]},\"id\":0}\n",
+      "headers": {
+        "Date": "Mon, 22 Jan 2018 19:14:51 GMT",
+        "Content-Type": "application/json",
+        "Transfer-Encoding": "chunked",
+        "Connection": "keep-alive",
+        "Set-Cookie": "__cfduid=d6d7bf13bce28ec88dc130f482acb91ab1516648491; expires=Tue, 22-Jan-19 19:14:51 GMT; path=/; domain=.poa.network; HttpOnly; Secure",
+        "Expect-CT": "max-age=604800, report-uri=\"https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct\"",
+        "Server": "cloudflare",
+        "CF-RAY": "3e14d6ad693128be-SJC"
+      },
+      "status_code": 200,
+      "type": "ok"
+    }
+  }
+]