Add a task to backfill old blocks

lib/explorer/skipped_blocks.ex

@@ -0,0 +1,38 @@
+defmodule Explorer.SkippedBlocks do
+  alias Explorer.Fetcher
+  alias Explorer.Block
+  alias Explorer.Repo
+  alias Ecto.Adapters.SQL
+  import Ecto.Query
+
+  @moduledoc false
+
+  @query """
+  SELECT missing_numbers.number AS missing_number
+  FROM generate_series(1, $1) missing_numbers(number)
+  LEFT OUTER JOIN blocks ON (blocks.number = missing_numbers.number)
+  WHERE blocks.id IS NULL;
+  """
+
+  @dialyzer {:nowarn_function, fetch: 0}
+  def fetch do
+    get_skipped_blocks()
+    |> Enum.map(&Integer.to_string/1)
+    |> Enum.map(&Fetcher.fetch/1)
+  end
+
+  def get_skipped_blocks do
+    last_block_number = get_last_block_number()
+    SQL.query!(Repo, @query, [last_block_number]).rows
+    |> Enum.map(&List.first/1)
+  end
+
+  def get_last_block_number do
+    block = Block
+    |> order_by(desc: :number)
+    |> limit(1)
+    |> Repo.all
+    |> List.first || %{number: 0}
+    block.number
+  end
+end

lib/mix/tasks/backfill.ex

@@ -0,0 +1,12 @@
+defmodule Mix.Tasks.Backfill do
+  use Mix.Task
+  alias Explorer.SkippedBlocks
+
+  @shortdoc "Backfill blocks from the chain."
+  @moduledoc false
+
+  def run(_) do
+    Mix.Task.run("app.start")
+    SkippedBlocks.fetch()
+  end
+end

test/explorer/skipped_blocks_test.exs

@@ -0,0 +1,53 @@
+defmodule Explorer.SkippedBlocksTest do
+  use Explorer.DataCase
+
+  alias Explorer.Block
+  alias Explorer.Repo
+  alias Explorer.SkippedBlocks
+
+  describe "fetch/0" do
+    test "inserts a missing block into the database" do
+      insert(:block, %{number: 2})
+      use_cassette "skipped_block_fetch" do
+        SkippedBlocks.fetch()
+
+        blocks = Block |> order_by(asc: :number) |> Repo.all |> Enum.map(fn(block) -> block.number end)
+
+        assert blocks == [1, 2]
+      end
+    end
+  end
+
+  describe "get_skipped_blocks/0 when there are no blocks" do
+    test "returns no blocks" do
+      assert SkippedBlocks.get_skipped_blocks() == []
+    end
+  end
+
+  describe "get_skipped_blocks/0 when there are no skipped blocks" do
+    test "returns no blocks" do
+      insert(:block, %{number: 1})
+      assert SkippedBlocks.get_skipped_blocks() == []
+    end
+  end
+
+  describe "get_skipped_blocks/0 when a block has been skipped" do
+    test "returns no blocks" do
+      insert(:block, %{number: 2})
+      assert SkippedBlocks.get_skipped_blocks() == [1]
+    end
+  end
+
+  describe "get_last_block_number/0 when there are no blocks" do
+    test "returns zero" do
+      assert SkippedBlocks.get_last_block_number() == 0
+    end
+  end
+
+  describe "get_last_block_number/0 when there is a block" do
+    test "returns the number of the block" do
+      insert(:block, %{number: 1})
+      assert SkippedBlocks.get_last_block_number() == 1
+    end
+  end
+end

test/mix/tasks/backfill_test.exs

@@ -0,0 +1,21 @@
+defmodule Scrape.Backfill do
+  use Explorer.DataCase
+  alias Explorer.Block
+  alias Explorer.Repo
+
+  test "backfills previous blocks" do
+    insert(:block, %{number: 2})
+
+    use_cassette "backfill" do
+      Mix.Tasks.Backfill.run([])
+
+      last_block = Block
+        |> order_by(asc: :number)
+        |> limit(1)
+        |> Repo.all
+        |> List.first
+
+      assert last_block.number == 1
+    end
+  end
+end

test/support/fixture/vcr_cassettes/backfill.json

@@ -0,0 +1,30 @@
+[
+  {
+    "request": {
+      "body": "{\"params\":[\"1\",true],\"method\":\"eth_getBlockByNumber\",\"jsonrpc\":\"2.0\",\"id\":2}",
+      "headers": {
+        "Content-Type": "application/json"
+      },
+      "method": "post",
+      "options": [],
+      "request_body": "",
+      "url": "https://sokol.poa.network:443"
+    },
+    "response": {
+      "binary": false,
+      "body": "{\"jsonrpc\":\"2.0\",\"result\":{\"author\":\"0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca\",\"difficulty\":\"0xffffffffffffffffffffffffedf58e45\",\"extraData\":\"0xd5830108048650617269747986312e32322e31826c69\",\"gasLimit\":\"0x66556d\",\"gasUsed\":\"0x0\",\"hash\":\"0x52c867bc0a91e573dc39300143c3bead7408d09d45bdb686749f02684ece72f3\",\"logsBloom\":\"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000\",\"miner\":\"0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca\",\"number\":\"0x1\",\"parentHash\":\"0x5b28c1bfd3a15230c9a46b399cd0f9a6920d432e85381cc6a140b06e8410112f\",\"receiptsRoot\":\"0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421\",\"sealFields\":[\"0x84120a71ba\",\"0xb8417a5887662f09ac4673af5850d28f3ad6550407b9c814ef563a13320f881b55ef03754f48f2dde027ad4a5abcabcc42780d9ebfc645f183e5252507d6a25bc2ec01\"],\"sha3Uncles\":\"0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347\",\"signature\":\"7a5887662f09ac4673af5850d28f3ad6550407b9c814ef563a13320f881b55ef03754f48f2dde027ad4a5abcabcc42780d9ebfc645f183e5252507d6a25bc2ec01\",\"size\":\"0x240\",\"stateRoot\":\"0xc196ad59d867542ef20b29df5f418d07dc7234f4bc3d25260526620b7958a8fb\",\"step\":\"302674362\",\"timestamp\":\"0x5a3438a2\",\"totalDifficulty\":\"0xffffffffffffffffffffffffedf78e45\",\"transactions\":[],\"transactionsRoot\":\"0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421\",\"uncles\":[]},\"id\":2}\n",
+      "headers": {
+        "Date": "Thu, 01 Feb 2018 19:05:58 GMT",
+        "Content-Type": "application/json",
+        "Transfer-Encoding": "chunked",
+        "Connection": "keep-alive",
+        "Set-Cookie": "__cfduid=d9d19faf0437d02b6374eec22299851e11517511958; expires=Fri, 01-Feb-19 19:05:58 GMT; path=/; domain=.poa.network; HttpOnly; Secure",
+        "Expect-CT": "max-age=604800, report-uri=\"https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct\"",
+        "Server": "cloudflare",
+        "CF-RAY": "3e672f6c0a1b9378-SJC"
+      },
+      "status_code": 200,
+      "type": "ok"
+    }
+  }
+]

test/support/fixture/vcr_cassettes/skipped_block_fetch.json

@@ -0,0 +1,30 @@
+[
+  {
+    "request": {
+      "body": "{\"params\":[\"1\",true],\"method\":\"eth_getBlockByNumber\",\"jsonrpc\":\"2.0\",\"id\":0}",
+      "headers": {
+        "Content-Type": "application/json"
+      },
+      "method": "post",
+      "options": [],
+      "request_body": "",
+      "url": "https://sokol.poa.network:443"
+    },
+    "response": {
+      "binary": false,
+      "body": "{\"jsonrpc\":\"2.0\",\"result\":{\"author\":\"0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca\",\"difficulty\":\"0xffffffffffffffffffffffffedf58e45\",\"extraData\":\"0xd5830108048650617269747986312e32322e31826c69\",\"gasLimit\":\"0x66556d\",\"gasUsed\":\"0x0\",\"hash\":\"0x52c867bc0a91e573dc39300143c3bead7408d09d45bdb686749f02684ece72f3\",\"logsBloom\":\"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000\",\"miner\":\"0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca\",\"number\":\"0x1\",\"parentHash\":\"0x5b28c1bfd3a15230c9a46b399cd0f9a6920d432e85381cc6a140b06e8410112f\",\"receiptsRoot\":\"0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421\",\"sealFields\":[\"0x84120a71ba\",\"0xb8417a5887662f09ac4673af5850d28f3ad6550407b9c814ef563a13320f881b55ef03754f48f2dde027ad4a5abcabcc42780d9ebfc645f183e5252507d6a25bc2ec01\"],\"sha3Uncles\":\"0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347\",\"signature\":\"7a5887662f09ac4673af5850d28f3ad6550407b9c814ef563a13320f881b55ef03754f48f2dde027ad4a5abcabcc42780d9ebfc645f183e5252507d6a25bc2ec01\",\"size\":\"0x240\",\"stateRoot\":\"0xc196ad59d867542ef20b29df5f418d07dc7234f4bc3d25260526620b7958a8fb\",\"step\":\"302674362\",\"timestamp\":\"0x5a3438a2\",\"totalDifficulty\":\"0xffffffffffffffffffffffffedf78e45\",\"transactions\":[],\"transactionsRoot\":\"0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421\",\"uncles\":[]},\"id\":0}\n",
+      "headers": {
+        "Date": "Thu, 01 Feb 2018 19:01:24 GMT",
+        "Content-Type": "application/json",
+        "Transfer-Encoding": "chunked",
+        "Connection": "keep-alive",
+        "Set-Cookie": "__cfduid=dd5b42dc11eb34648ef3c5b13cd49fe5c1517511684; expires=Fri, 01-Feb-19 19:01:24 GMT; path=/; domain=.poa.network; HttpOnly; Secure",
+        "Expect-CT": "max-age=604800, report-uri=\"https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct\"",
+        "Server": "cloudflare",
+        "CF-RAY": "3e6728ba88d192b2-SJC"
+      },
+      "status_code": 200,
+      "type": "ok"
+    }
+  }
+]