From 7c0cf2c2c460e91dfca7788210af9c2f65dd6d0d Mon Sep 17 00:00:00 2001 From: Doc Ritezel Date: Thu, 1 Feb 2018 11:07:00 -0800 Subject: [PATCH] Add a task to backfill old blocks --- lib/explorer/skipped_blocks.ex | 38 +++++++++++++ lib/mix/tasks/backfill.ex | 12 +++++ test/explorer/skipped_blocks_test.exs | 53 +++++++++++++++++++ test/mix/tasks/backfill_test.exs | 21 ++++++++ .../fixture/vcr_cassettes/backfill.json | 30 +++++++++++ .../vcr_cassettes/skipped_block_fetch.json | 30 +++++++++++ 6 files changed, 184 insertions(+) create mode 100644 lib/explorer/skipped_blocks.ex create mode 100644 lib/mix/tasks/backfill.ex create mode 100644 test/explorer/skipped_blocks_test.exs create mode 100644 test/mix/tasks/backfill_test.exs create mode 100644 test/support/fixture/vcr_cassettes/backfill.json create mode 100644 test/support/fixture/vcr_cassettes/skipped_block_fetch.json diff --git a/lib/explorer/skipped_blocks.ex b/lib/explorer/skipped_blocks.ex new file mode 100644 index 0000000000..db2479de64 --- /dev/null +++ b/lib/explorer/skipped_blocks.ex @@ -0,0 +1,38 @@ +defmodule Explorer.SkippedBlocks do + alias Explorer.Fetcher + alias Explorer.Block + alias Explorer.Repo + alias Ecto.Adapters.SQL + import Ecto.Query + + @moduledoc false + + @query """ + SELECT missing_numbers.number AS missing_number + FROM generate_series(1, $1) missing_numbers(number) + LEFT OUTER JOIN blocks ON (blocks.number = missing_numbers.number) + WHERE blocks.id IS NULL; + """ + + @dialyzer {:nowarn_function, fetch: 0} + def fetch do + get_skipped_blocks() + |> Enum.map(&Integer.to_string/1) + |> Enum.map(&Fetcher.fetch/1) + end + + def get_skipped_blocks do + last_block_number = get_last_block_number() + SQL.query!(Repo, @query, [last_block_number]).rows + |> Enum.map(&List.first/1) + end + + def get_last_block_number do + block = Block + |> order_by(desc: :number) + |> limit(1) + |> Repo.all + |> List.first || %{number: 0} + block.number + end +end diff --git a/lib/mix/tasks/backfill.ex b/lib/mix/tasks/backfill.ex new file mode 100644 index 0000000000..b4e52b3105 --- /dev/null +++ b/lib/mix/tasks/backfill.ex @@ -0,0 +1,12 @@ +defmodule Mix.Tasks.Backfill do + use Mix.Task + alias Explorer.SkippedBlocks + + @shortdoc "Backfill blocks from the chain." + @moduledoc false + + def run(_) do + Mix.Task.run("app.start") + SkippedBlocks.fetch() + end +end diff --git a/test/explorer/skipped_blocks_test.exs b/test/explorer/skipped_blocks_test.exs new file mode 100644 index 0000000000..5d0efa6f59 --- /dev/null +++ b/test/explorer/skipped_blocks_test.exs @@ -0,0 +1,53 @@ +defmodule Explorer.SkippedBlocksTest do + use Explorer.DataCase + + alias Explorer.Block + alias Explorer.Repo + alias Explorer.SkippedBlocks + + describe "fetch/0" do + test "inserts a missing block into the database" do + insert(:block, %{number: 2}) + use_cassette "skipped_block_fetch" do + SkippedBlocks.fetch() + + blocks = Block |> order_by(asc: :number) |> Repo.all |> Enum.map(fn(block) -> block.number end) + + assert blocks == [1, 2] + end + end + end + + describe "get_skipped_blocks/0 when there are no blocks" do + test "returns no blocks" do + assert SkippedBlocks.get_skipped_blocks() == [] + end + end + + describe "get_skipped_blocks/0 when there are no skipped blocks" do + test "returns no blocks" do + insert(:block, %{number: 1}) + assert SkippedBlocks.get_skipped_blocks() == [] + end + end + + describe "get_skipped_blocks/0 when a block has been skipped" do + test "returns no blocks" do + insert(:block, %{number: 2}) + assert SkippedBlocks.get_skipped_blocks() == [1] + end + end + + describe "get_last_block_number/0 when there are no blocks" do + test "returns zero" do + assert SkippedBlocks.get_last_block_number() == 0 + end + end + + describe "get_last_block_number/0 when there is a block" do + test "returns the number of the block" do + insert(:block, %{number: 1}) + assert SkippedBlocks.get_last_block_number() == 1 + end + end +end diff --git a/test/mix/tasks/backfill_test.exs b/test/mix/tasks/backfill_test.exs new file mode 100644 index 0000000000..7c321cc8f0 --- /dev/null +++ b/test/mix/tasks/backfill_test.exs @@ -0,0 +1,21 @@ +defmodule Scrape.Backfill do + use Explorer.DataCase + alias Explorer.Block + alias Explorer.Repo + + test "backfills previous blocks" do + insert(:block, %{number: 2}) + + use_cassette "backfill" do + Mix.Tasks.Backfill.run([]) + + last_block = Block + |> order_by(asc: :number) + |> limit(1) + |> Repo.all + |> List.first + + assert last_block.number == 1 + end + end +end diff --git a/test/support/fixture/vcr_cassettes/backfill.json b/test/support/fixture/vcr_cassettes/backfill.json new file mode 100644 index 0000000000..0468d77beb --- /dev/null +++ b/test/support/fixture/vcr_cassettes/backfill.json @@ -0,0 +1,30 @@ +[ + { + "request": { + "body": "{\"params\":[\"1\",true],\"method\":\"eth_getBlockByNumber\",\"jsonrpc\":\"2.0\",\"id\":2}", + "headers": { + "Content-Type": "application/json" + }, + "method": "post", + "options": [], + "request_body": "", + "url": "https://sokol.poa.network:443" + }, + "response": { + "binary": false, + "body": "{\"jsonrpc\":\"2.0\",\"result\":{\"author\":\"0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca\",\"difficulty\":\"0xffffffffffffffffffffffffedf58e45\",\"extraData\":\"0xd5830108048650617269747986312e32322e31826c69\",\"gasLimit\":\"0x66556d\",\"gasUsed\":\"0x0\",\"hash\":\"0x52c867bc0a91e573dc39300143c3bead7408d09d45bdb686749f02684ece72f3\",\"logsBloom\":\"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000\",\"miner\":\"0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca\",\"number\":\"0x1\",\"parentHash\":\"0x5b28c1bfd3a15230c9a46b399cd0f9a6920d432e85381cc6a140b06e8410112f\",\"receiptsRoot\":\"0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421\",\"sealFields\":[\"0x84120a71ba\",\"0xb8417a5887662f09ac4673af5850d28f3ad6550407b9c814ef563a13320f881b55ef03754f48f2dde027ad4a5abcabcc42780d9ebfc645f183e5252507d6a25bc2ec01\"],\"sha3Uncles\":\"0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347\",\"signature\":\"7a5887662f09ac4673af5850d28f3ad6550407b9c814ef563a13320f881b55ef03754f48f2dde027ad4a5abcabcc42780d9ebfc645f183e5252507d6a25bc2ec01\",\"size\":\"0x240\",\"stateRoot\":\"0xc196ad59d867542ef20b29df5f418d07dc7234f4bc3d25260526620b7958a8fb\",\"step\":\"302674362\",\"timestamp\":\"0x5a3438a2\",\"totalDifficulty\":\"0xffffffffffffffffffffffffedf78e45\",\"transactions\":[],\"transactionsRoot\":\"0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421\",\"uncles\":[]},\"id\":2}\n", + "headers": { + "Date": "Thu, 01 Feb 2018 19:05:58 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "Set-Cookie": "__cfduid=d9d19faf0437d02b6374eec22299851e11517511958; expires=Fri, 01-Feb-19 19:05:58 GMT; path=/; domain=.poa.network; HttpOnly; Secure", + "Expect-CT": "max-age=604800, report-uri=\"https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct\"", + "Server": "cloudflare", + "CF-RAY": "3e672f6c0a1b9378-SJC" + }, + "status_code": 200, + "type": "ok" + } + } +] \ No newline at end of file diff --git a/test/support/fixture/vcr_cassettes/skipped_block_fetch.json b/test/support/fixture/vcr_cassettes/skipped_block_fetch.json new file mode 100644 index 0000000000..085057719d --- /dev/null +++ b/test/support/fixture/vcr_cassettes/skipped_block_fetch.json @@ -0,0 +1,30 @@ +[ + { + "request": { + "body": "{\"params\":[\"1\",true],\"method\":\"eth_getBlockByNumber\",\"jsonrpc\":\"2.0\",\"id\":0}", + "headers": { + "Content-Type": "application/json" + }, + "method": "post", + "options": [], + "request_body": "", + "url": "https://sokol.poa.network:443" + }, + "response": { + "binary": false, + "body": "{\"jsonrpc\":\"2.0\",\"result\":{\"author\":\"0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca\",\"difficulty\":\"0xffffffffffffffffffffffffedf58e45\",\"extraData\":\"0xd5830108048650617269747986312e32322e31826c69\",\"gasLimit\":\"0x66556d\",\"gasUsed\":\"0x0\",\"hash\":\"0x52c867bc0a91e573dc39300143c3bead7408d09d45bdb686749f02684ece72f3\",\"logsBloom\":\"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000\",\"miner\":\"0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca\",\"number\":\"0x1\",\"parentHash\":\"0x5b28c1bfd3a15230c9a46b399cd0f9a6920d432e85381cc6a140b06e8410112f\",\"receiptsRoot\":\"0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421\",\"sealFields\":[\"0x84120a71ba\",\"0xb8417a5887662f09ac4673af5850d28f3ad6550407b9c814ef563a13320f881b55ef03754f48f2dde027ad4a5abcabcc42780d9ebfc645f183e5252507d6a25bc2ec01\"],\"sha3Uncles\":\"0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347\",\"signature\":\"7a5887662f09ac4673af5850d28f3ad6550407b9c814ef563a13320f881b55ef03754f48f2dde027ad4a5abcabcc42780d9ebfc645f183e5252507d6a25bc2ec01\",\"size\":\"0x240\",\"stateRoot\":\"0xc196ad59d867542ef20b29df5f418d07dc7234f4bc3d25260526620b7958a8fb\",\"step\":\"302674362\",\"timestamp\":\"0x5a3438a2\",\"totalDifficulty\":\"0xffffffffffffffffffffffffedf78e45\",\"transactions\":[],\"transactionsRoot\":\"0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421\",\"uncles\":[]},\"id\":0}\n", + "headers": { + "Date": "Thu, 01 Feb 2018 19:01:24 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "Set-Cookie": "__cfduid=dd5b42dc11eb34648ef3c5b13cd49fe5c1517511684; expires=Fri, 01-Feb-19 19:01:24 GMT; path=/; domain=.poa.network; HttpOnly; Secure", + "Expect-CT": "max-age=604800, report-uri=\"https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct\"", + "Server": "cloudflare", + "CF-RAY": "3e6728ba88d192b2-SJC" + }, + "status_code": 200, + "type": "ok" + } + } +] \ No newline at end of file