Switch to a worker model for job scheduling and execution

pull/2/head
Doc Ritezel 7 years ago
parent 4f326808cd
commit f0ace70d9b
  1. 1
      Procfile
  2. 10
      bin/deploy
  3. 1
      circle.yml
  4. 15
      config/config.exs
  5. 6
      config/dev.exs
  6. 8
      config/prod.exs
  7. 29
      lib/explorer/application.ex
  8. 13
      lib/explorer/block.ex
  9. 40
      lib/explorer/fetcher.ex
  10. 16
      lib/explorer/latest_block.ex
  11. 33
      lib/explorer/skipped_blocks.ex
  12. 17
      lib/explorer/workers/import_block.ex
  13. 16
      lib/explorer/workers/import_skipped_blocks.ex
  14. 34
      lib/explorer_web/router.ex
  15. 12
      lib/mix/tasks/backfill.ex
  16. 18
      lib/mix/tasks/exq.start.ex
  17. 13
      lib/mix/tasks/scrape.ex
  18. 13
      mix.exs
  19. 4
      mix.lock
  20. 15
      test/explorer/block_test.exs
  21. 31
      test/explorer/latest_block_test.exs
  22. 69
      test/explorer/skipped_blocks_test.exs
  23. 38
      test/explorer/workers/import_block_test.exs
  24. 23
      test/explorer/workers/import_skipped_blocks_test.exs
  25. 11
      test/explorer_web/features/exq_test.exs
  26. 21
      test/mix/tasks/backfill_test.exs
  27. 19
      test/mix/tasks/scrape_test.exs
  28. 1
      test/support/fixture/vcr_cassettes/backfill_skipped_blocks_perform_1.json
  29. 30
      test/support/fixture/vcr_cassettes/import_block_perform_1_duplicate.json
  30. 30
      test/support/fixture/vcr_cassettes/import_block_perform_1_earliest.json
  31. 30
      test/support/fixture/vcr_cassettes/import_block_perform_1_integer.json
  32. 30
      test/support/fixture/vcr_cassettes/import_block_perform_1_string.json
  33. 1
      test/support/fixture/vcr_cassettes/import_skipped_blocks_perform_1.json

@ -1 +1,2 @@
web: mix phx.server
worker: mix exq.start

@ -12,25 +12,31 @@ then
git fetch heroku
fi
WORKER_COUNT=$(heroku ps | grep 'worker\.' | wc -l)
if ! git diff HEAD heroku/master --exit-code -- priv/repo
then
if heroku features --app $HEROKU_APPLICATION | grep '\[+\] preboot'
then
heroku features:disable preboot --app $HEROKU_APPLICATION
heroku maintenance:on --app $HEROKU_APPLICATION
heroku scale worker=0 --app $HEROKU_APPLICATION
heroku pg:killall --app $HEROKU_APPLICATION
git push heroku $CIRCLE_SHA1:refs/heads/master
heroku pg:backups capture --app $HEROKU_APPLICATION
heroku run "POOL_SIZE=2 mix ecto.migrate" --app $HEROKU_APPLICATION
heroku scale worker=$WORKER_COUNT --app $HEROKU_APPLICATION
heroku restart --app $HEROKU_APPLICATION
heroku maintenance:off --app $HEROKU_APPLICATION
heroku features:enable preboot --app $HEROKU_APPLICATION
else
heroku maintenance:on --app $HEROKU_APPLICATION
heroku scale worker=0 --app $HEROKU_APPLICATION
heroku pg:killall --app $HEROKU_APPLICATION
git push heroku $CIRCLE_SHA1:refs/heads/master
heroku pg:backups capture --app $HEROKU_APPLICATION
heroku run "POOL_SIZE=2 mix ecto.migrate" --app $HEROKU_APPLICATION
heroku scale worker=$WORKER_COUNT --app $HEROKU_APPLICATION
heroku restart --app $HEROKU_APPLICATION
heroku maintenance:off --app $HEROKU_APPLICATION
fi
@ -38,3 +44,7 @@ else
git push heroku $CIRCLE_SHA1:refs/heads/master
heroku pg:backups capture --app $HEROKU_APPLICATION
fi
set +x
heroku config:set ADMIN_PASSWORD=$(mix phx.gen.secret) --app $HEROKU_APPLICATION

@ -10,6 +10,7 @@ machine:
version: 9.4.0
services:
- postgresql
- redis
pre:
- mkdir -p $CIRCLE_TEST_REPORTS/exunit
- mkdir -p $CIRCLE_TEST_REPORTS/eslint

@ -18,6 +18,9 @@ config :explorer, ExplorerWeb.Endpoint,
render_errors: [view: ExplorerWeb.ErrorView, accepts: ~w(html json)],
pubsub: [name: Explorer.PubSub, adapter: Phoenix.PubSub.PG2]
config :explorer, Explorer.Integrations.EctoLogger,
query_time_ms_threshold: 2_000
# Configures Elixir's Logger
config :logger, :console,
format: "$time $metadata[$level] $message\n",
@ -33,6 +36,18 @@ config :ethereumex,
locales: ["en"],
gettext: ExplorerWeb.Gettext
config :exq,
host: "localhost",
port: 6379,
namespace: "exq",
start_on_application: false,
scheduler_enable: true,
shutdown_timeout: 5000,
max_retries: 10
config :exq_ui,
server: false
# Import environment specific config. This must remain at the bottom
# of this file so it overrides the configuration defined above.
import_config "#{Mix.env}.exs"

@ -58,7 +58,11 @@ config :explorer, Explorer.Repo,
# Configure Quantum
config :explorer, Explorer.Scheduler,
jobs: [
{"* * * * *", {Mix.Tasks.Scrape, :run, [:ok]}},
[schedule: {:extended, "*/5 * * * * *"}, task: {Explorer.Workers.ImportBlock, :perform_later, ["latest"]}],
[schedule: {:extended, "*/15 * * * * *"}, task: {Explorer.Workers.ImportSkippedBlocks, :perform_later, [1]}],
]
config :exq,
concurrency: 4
import_config "dev.secret.exs"

@ -46,5 +46,11 @@ config :ethereumex,
# Configure Quantum
config :explorer, Explorer.Scheduler,
jobs: [
{"@secondly", {Mix.Tasks.Scrape, :run, [[]]}}
[schedule: {:extended, "* * * * * *"}, task: {Explorer.Workers.ImportBlock, :perform_later, ["latest"]}],
[schedule: {:extended, "*/15 * * * * *"}, task: {Explorer.Workers.ImportSkippedBlocks, :perform_later, [5]}],
]
# Configure Exq
config :exq,
url: {:system, "REDIS_URL"},
concurrency: 10

@ -8,22 +8,10 @@ defmodule Explorer.Application do
# See https://hexdocs.pm/elixir/Application.html
# for more information on OTP Applications
def start(_type, _args) do
import Supervisor.Spec
# Define workers and child supervisors to be supervised
children = [
# Start the Ecto repository
supervisor(Explorer.Repo, []),
# Start the endpoint when the application starts
supervisor(ExplorerWeb.Endpoint, []),
# Start your own worker by calling: Explorer.Worker.start_link(a, b, c)
worker(Explorer.Scheduler, []),
]
# See https://hexdocs.pm/elixir/Supervisor.html
# for other strategies and supported options
opts = [strategy: :one_for_one, name: Explorer.Supervisor]
Supervisor.start_link(children, opts)
Supervisor.start_link(children(Mix.env), opts)
end
# Tell Phoenix to update the endpoint configuration
@ -33,4 +21,19 @@ defmodule Explorer.Application do
Endpoint.config_change(changed, removed)
:ok
end
defp children(:test), do: children()
defp children(_) do
import Supervisor.Spec
exq_options = [] |> Keyword.put(:mode, :enqueuer)
[supervisor(Exq, [exq_options]) | children()]
end
defp children do
import Supervisor.Spec
[
supervisor(Explorer.Repo, []),
supervisor(ExplorerWeb.Endpoint, []),
]
end
end

@ -1,10 +1,11 @@
defmodule Explorer.Block do
@moduledoc false
use Ecto.Schema
import Ecto.Changeset
import Ecto.Query
alias Explorer.Block
@moduledoc false
@timestamps_opts [type: Timex.Ecto.DateTime,
autogenerate: {Timex.Ecto.DateTime, :autogenerate, []}]
@ -38,4 +39,12 @@ defmodule Explorer.Block do
|> update_change(:hash, &String.downcase/1)
|> unique_constraint(:hash)
end
def null do
%Block{number: -1}
end
def latest(query) do
query |> order_by(desc: :number)
end
end

@ -6,17 +6,17 @@ defmodule Explorer.Fetcher do
alias Explorer.Repo
alias Explorer.ToAddress
alias Explorer.Transaction
import Ethereumex.HttpClient, only: [eth_get_block_by_number: 2]
@dialyzer {:nowarn_function, fetch: 1}
def fetch(block_number) do
raw_block = block_number |> download_block
Repo.transaction fn ->
raw_block
|> extract_block
|> prepare_block
|> Repo.insert!
|> Repo.insert_or_update!
|> extract_transactions(raw_block["transactions"])
end
end
@ -49,12 +49,13 @@ defmodule Explorer.Fetcher do
end)
end
def create_transaction(block, transaction) do
%Transaction{}
|> Transaction.changeset(extract_transaction(block, transaction))
|> Repo.insert!
|> create_from_address(transaction["from"])
|> create_to_address(transaction["to"] || transaction["creates"])
def create_transaction(block, changes) do
transaction = Repo.get_by(Transaction, hash: changes["hash"]) || %Transaction{}
transaction
|> Transaction.changeset(extract_transaction(block, changes))
|> Repo.insert_or_update!
|> create_from_address(changes["from"])
|> create_to_address(changes["to"] || changes["creates"])
end
def extract_transaction(block, transaction) do
@ -77,28 +78,31 @@ defmodule Explorer.Fetcher do
def create_to_address(transaction, hash) do
address = Address.find_or_create_by_hash(hash)
attrs = %{transaction_id: transaction.id, address_id: address.id}
changes = %{transaction_id: transaction.id, address_id: address.id}
%ToAddress{}
|> ToAddress.changeset(attrs)
|> Repo.insert
to_address = Repo.get_by(ToAddress, changes) || %ToAddress{}
to_address
|> ToAddress.changeset(changes)
|> Repo.insert_or_update!
transaction
end
def create_from_address(transaction, hash) do
address = Address.find_or_create_by_hash(hash)
attrs = %{transaction_id: transaction.id, address_id: address.id}
changes = %{transaction_id: transaction.id, address_id: address.id}
%FromAddress{}
|> FromAddress.changeset(attrs)
|> Repo.insert
from_address = Repo.get_by(FromAddress, changes) || %FromAddress{}
from_address
|> FromAddress.changeset(changes)
|> Repo.insert_or_update!
transaction
end
def prepare_block(block) do
Block.changeset(%Block{}, block)
def prepare_block(changes) do
block = Repo.get_by(Block, hash: changes.hash) || %Block{}
Block.changeset(block, changes)
end
def decode_integer_field(hex) do

@ -1,16 +0,0 @@
defmodule Explorer.LatestBlock do
alias Explorer.Fetcher
import Ethereumex.HttpClient, only: [eth_block_number: 0]
@moduledoc false
@dialyzer {:nowarn_function, fetch: 0}
def fetch do
get_latest_block() |> Fetcher.fetch
end
def get_latest_block do
{:ok, block_number} = eth_block_number()
block_number
end
end

@ -1,38 +1,29 @@
defmodule Explorer.SkippedBlocks do
alias Explorer.Fetcher
alias Explorer.Block
alias Explorer.Repo
alias Ecto.Adapters.SQL
import Ecto.Query
import Ecto.Query, only: [limit: 2]
@moduledoc false
@query """
SELECT missing_numbers.number AS missing_number
FROM generate_series($1, 1, -1) missing_numbers(number)
FROM generate_series($1, 0, -1) missing_numbers(number)
LEFT OUTER JOIN blocks ON (blocks.number = missing_numbers.number)
WHERE blocks.id IS NULL;
WHERE blocks.id IS NULL
LIMIT $2;
"""
@dialyzer {:nowarn_function, fetch: 0}
def fetch do
get_skipped_blocks()
|> Enum.map(&Integer.to_string/1)
|> Enum.map(&Fetcher.fetch/1)
end
def get_skipped_blocks do
last_block_number = get_last_block_number()
SQL.query!(Repo, @query, [last_block_number]).rows
def first, do: first(1)
def first(count) do
SQL.query!(Repo, @query, [latest_block_number(), count]).rows
|> Enum.map(&List.first/1)
|> Enum.map(&Integer.to_string/1)
end
def get_last_block_number do
block = Block
|> order_by(desc: :number)
|> limit(1)
|> Repo.all
|> List.first || %{number: 0}
block.number
def latest_block_number do
(Block |> Block.latest |> limit(1) |> Repo.one || Block.null)
|> Map.fetch!(:number)
end
end

@ -0,0 +1,17 @@
defmodule Explorer.Workers.ImportBlock do
alias Explorer.Fetcher
@moduledoc "Imports blocks by web3 conventions."
@dialyzer {:nowarn_function, perform: 1}
def perform(number) do
Fetcher.fetch("#{number}")
end
@dialyzer {:nowarn_function, perform: 0}
def perform, do: perform("latest")
def perform_later(number) do
Exq.enqueue(Exq.Enqueuer, "default", __MODULE__, [number])
end
end

@ -0,0 +1,16 @@
defmodule Explorer.Workers.ImportSkippedBlocks do
alias Explorer.SkippedBlocks
alias Explorer.Workers.ImportBlock
@moduledoc "Imports skipped blocks."
def perform, do: perform(1)
def perform(count) do
count |> SkippedBlocks.first |> Enum.map(&ImportBlock.perform_later/1)
end
def perform_later, do: perform_later(1)
def perform_later(count) do
Exq.enqueue(Exq.Enqueuer, "default", __MODULE__, [count])
end
end

@ -15,21 +15,51 @@ defmodule ExplorerWeb.Router do
font-src 'self' 'unsafe-inline' 'unsafe-eval' data:;\
"
}
if Mix.env != :prod, do: plug Jasmine, js_files: ["js/test.js"], css_files: ["css/test.css"]
end
pipeline :set_locale do
plug SetLocale, gettext: ExplorerWeb.Gettext, default_locale: "en"
end
pipeline :exq do
plug :accepts, ["html"]
plug :fetch_session
plug :fetch_flash
plug :put_secure_browser_headers, %{
"content-security-policy" => "\
default-src 'self';\
script-src 'self' 'unsafe-inline';\
font-src 'self' fonts.gstatic.com;\
style-src 'self' 'unsafe-inline' fonts.googleapis.com;\
"
}
plug ExqUi.RouterPlug, namespace: "exq"
end
pipeline :jasmine do
if Mix.env != :prod, do: plug Jasmine, js_files: ["js/test.js"], css_files: ["css/test.css"]
end
pipeline :api do
plug :accepts, ["json"]
end
scope "/exq", ExqUi do
pipe_through :exq
forward "/", RouterPlug.Router, :index
end
scope "/", ExplorerWeb do
pipe_through :browser
pipe_through :jasmine
pipe_through :set_locale
resources "/", ChainController, only: [:show], singleton: true, as: :chain
end
scope "/:locale", ExplorerWeb do
pipe_through :browser # Use the default browser stack
pipe_through :browser
pipe_through :jasmine
pipe_through :set_locale
resources "/", ChainController, only: [:show], singleton: true, as: :chain
resources "/blocks", BlockController, only: [:index, :show]
resources "/transactions", TransactionController, only: [:index, :show]

@ -1,12 +0,0 @@
defmodule Mix.Tasks.Backfill do
use Mix.Task
alias Explorer.SkippedBlocks
@shortdoc "Backfill blocks from the chain."
@moduledoc false
def run(_) do
Mix.Task.run("app.start")
SkippedBlocks.fetch()
end
end

@ -0,0 +1,18 @@
defmodule Mix.Tasks.Exq.Start do
alias Explorer.Repo
alias Explorer.Scheduler
use Mix.Task
@moduledoc "Starts the Exq worker"
def run(_args) do
[:postgrex, :ecto, :ethereumex, :tzdata]
|> Enum.each(&Application.ensure_all_started/1)
Repo.start_link()
Exq.start_link(mode: :default)
Scheduler.start_link()
IO.puts "Started Exq"
:timer.sleep(:infinity)
end
end

@ -1,13 +0,0 @@
defmodule Mix.Tasks.Scrape do
use Mix.Task
alias Explorer.LatestBlock
@shortdoc "Scrape the blockchain."
@moduledoc false
@dialyzer {:nowarn_function, run: 1}
def run(_) do
Mix.Task.run("app.start")
LatestBlock.fetch()
end
end

@ -35,11 +35,12 @@ defmodule Explorer.Mixfile do
defp elixirc_paths, do: ["lib"]
# Specifies extra applications to start per environment
defp extra_applications(:prod), do: [:phoenix_pubsub_redis, :new_relixir | extra_applications()]
defp extra_applications(:prod), do: [:phoenix_pubsub_redis, :new_relixir, :exq, :exq_ui | extra_applications()]
defp extra_applications(:dev), do: [:exq, :exq_ui | extra_applications()]
defp extra_applications(_), do: extra_applications()
defp extra_applications, do: [
:scrivener_ecto, :scrivener_html, :ex_cldr, :ex_jasmine, :ethereumex,
:timex, :timex_ecto, :set_locale, :logger, :runtime_tools
:timex, :timex_ecto, :crontab, :set_locale, :logger, :runtime_tools
]
# Specifies your project dependencies.
@ -49,16 +50,20 @@ defmodule Explorer.Mixfile do
[
{:cowboy, "~> 1.0"},
{:credo, "~> 0.8", only: [:dev, :test], runtime: false},
{:crontab, "~> 1.1"},
{:dialyxir, "~> 0.5", only: [:dev, :test], runtime: false},
{:ethereumex, github: "exthereum/ethereumex", commit: "262f1d81ae163ffb46e127283658249dac1c8318"}, # Waiting for this version to be pushed to Hex.
{:ex_cldr_numbers, "~> 1.0"},
{:ex_cldr_units, "~> 1.0"},
{:ex_jasmine, github: "minifast/ex_jasmine", branch: "master"},
{:ex_machina, "~> 2.1", only: [:test]},
{:exq, "~> 0.9.1"},
{:exq_ui, "~> 0.9.0"},
{:exvcr, "~> 0.8", only: :test},
{:gettext, "~> 0.11"},
{:ex_jasmine, github: "minifast/ex_jasmine", branch: "master"},
{:junit_formatter, ">= 0.0.0"},
{:junit_formatter, ">= 0.0.0", only: [:test], runtime: false},
{:math, "~> 0.3.0"},
{:mock, "~> 0.3.0", only: [:test], runtime: false},
{:new_relixir, "~> 0.4.0", only: [:prod]},
{:phoenix, "~> 1.3.0"},
{:phoenix_ecto, "~> 3.2"},

@ -19,6 +19,8 @@
"ex_machina": {:hex, :ex_machina, "2.1.0", "4874dc9c78e7cf2d429f24dc3c4005674d4e4da6a08be961ffccc08fb528e28b", [:mix], [{:ecto, "~> 2.1", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"},
"exactor": {:hex, :exactor, "2.2.4", "5efb4ddeb2c48d9a1d7c9b465a6fffdd82300eb9618ece5d34c3334d5d7245b1", [:mix], [], "hexpm"},
"exjsx": {:hex, :exjsx, "4.0.0", "60548841e0212df401e38e63c0078ec57b33e7ea49b032c796ccad8cde794b5c", [:mix], [{:jsx, "~> 2.8.0", [hex: :jsx, repo: "hexpm", optional: false]}], "hexpm"},
"exq": {:hex, :exq, "0.9.1", "77ee12c117411ddddd9810aec714eaa704fb8f327949551d9efaea19606122c3", [:mix], [{:poison, ">= 1.2.0 or ~> 2.0", [hex: :poison, repo: "hexpm", optional: false]}, {:redix, ">= 0.5.0", [hex: :redix, repo: "hexpm", optional: false]}, {:uuid, ">= 1.1.0", [hex: :uuid, repo: "hexpm", optional: false]}], "hexpm"},
"exq_ui": {:hex, :exq_ui, "0.9.0", "e97e9fa9009f30d2926b51a166e40a3a521e83f61f3f333fede8335b2aa57f09", [:mix], [{:cowboy, "~> 1.0", [hex: :cowboy, repo: "hexpm", optional: false]}, {:exq, "~> 0.9", [hex: :exq, repo: "hexpm", optional: false]}, {:plug, ">= 0.8.1 and < 2.0.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"},
"exvcr": {:hex, :exvcr, "0.9.1", "31e3936a790a14bf56b31b6b276577076a5ef8afd9b2d53ba3ff8bb647d45613", [:mix], [{:exactor, "~> 2.2", [hex: :exactor, repo: "hexpm", optional: false]}, {:exjsx, "~> 4.0", [hex: :exjsx, repo: "hexpm", optional: false]}, {:httpoison, "~> 0.13", [hex: :httpoison, repo: "hexpm", optional: true]}, {:httpotion, "~> 3.0", [hex: :httpotion, repo: "hexpm", optional: true]}, {:ibrowse, "~> 4.4", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:meck, "~> 0.8.8", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm"},
"file_system": {:hex, :file_system, "0.2.4", "f0bdda195c0e46e987333e986452ec523aed21d784189144f647c43eaf307064", [:mix], [], "hexpm"},
"gen_stage": {:hex, :gen_stage, "0.12.2", "e0e347cbb1ceb5f4e68a526aec4d64b54ad721f0a8b30aa9d28e0ad749419cbb", [:mix], [], "hexpm"},
@ -33,6 +35,7 @@
"metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm"},
"mime": {:hex, :mime, "1.2.0", "78adaa84832b3680de06f88f0997e3ead3b451a440d183d688085be2d709b534", [:mix], [], "hexpm"},
"mimerl": {:hex, :mimerl, "1.0.2", "993f9b0e084083405ed8252b99460c4f0563e41729ab42d9074fd5e52439be88", [:rebar3], [], "hexpm"},
"mock": {:hex, :mock, "0.3.1", "994f00150f79a0ea50dc9d86134cd9ebd0d177ad60bd04d1e46336cdfdb98ff9", [:mix], [{:meck, "~> 0.8.8", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm"},
"new_relixir": {:hex, :new_relixir, "0.4.0", "32e15de5dc21bd9823040d3d7fe62cd931755a06de2163e02b79a703179ea102", [:mix], [{:ecto, "~> 2.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:hackney, "~> 1.10", [hex: :hackney, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.3", [hex: :phoenix, repo: "hexpm", optional: false]}], "hexpm"},
"phoenix": {:hex, :phoenix, "1.3.0", "1c01124caa1b4a7af46f2050ff11b267baa3edb441b45dbf243e979cd4c5891b", [:mix], [{:cowboy, "~> 1.0", [hex: :cowboy, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.0", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.3.3 or ~> 1.4", [hex: :plug, repo: "hexpm", optional: false]}, {:poison, "~> 2.2 or ~> 3.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm"},
"phoenix_ecto": {:hex, :phoenix_ecto, "3.3.0", "702f6e164512853d29f9d20763493f2b3bcfcb44f118af2bc37bb95d0801b480", [:mix], [{:ecto, "~> 2.1", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.9", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"},
@ -59,4 +62,5 @@
"timex_ecto": {:hex, :timex_ecto, "3.2.1", "461140751026e1ca03298fab628f78ab189e78784175f5e301eefa034ee530aa", [:mix], [{:ecto, "~> 2.2", [hex: :ecto, repo: "hexpm", optional: false]}, {:timex, "~> 3.1", [hex: :timex, repo: "hexpm", optional: false]}], "hexpm"},
"tzdata": {:hex, :tzdata, "0.5.16", "13424d3afc76c68ff607f2df966c0ab4f3258859bbe3c979c9ed1606135e7352", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"},
"unicode_util_compat": {:hex, :unicode_util_compat, "0.3.1", "a1f612a7b512638634a603c8f401892afbf99b8ce93a45041f8aaca99cadb85e", [:rebar3], [], "hexpm"},
"uuid": {:hex, :uuid, "1.1.8", "e22fc04499de0de3ed1116b770c7737779f226ceefa0badb3592e64d5cfb4eb9", [:mix], [], "hexpm"},
"wallaby": {:hex, :wallaby, "0.19.2", "358bbff251e3555e02566280d1795132da792969dd58ff89963536d013058719", [:mix], [{:httpoison, "~> 0.12", [hex: :httpoison, repo: "hexpm", optional: false]}, {:poison, ">= 1.4.0", [hex: :poison, repo: "hexpm", optional: false]}, {:poolboy, "~> 1.5", [hex: :poolboy, repo: "hexpm", optional: false]}], "hexpm"}}

@ -2,6 +2,7 @@ defmodule Explorer.BlockTest do
use Explorer.DataCase
alias Explorer.Block
import Ecto.Query, only: [order_by: 2]
describe "changeset/2" do
test "with valid attributes" do
@ -28,4 +29,18 @@ defmodule Explorer.BlockTest do
assert changeset.errors == [hash: {"has already been taken", []}]
end
end
describe "null/0" do
test "returns a block with a number of 0" do
assert Block.null.number === -1
end
end
describe "latest/1" do
test "returns the blocks sorted by number" do
insert(:block, number: 1)
insert(:block, number: 5)
assert Block |> Block.latest |> Repo.all == Block |> order_by(desc: :number) |> Repo.all
end
end
end

@ -1,31 +0,0 @@
defmodule Explorer.LatestBlockTest do
use Explorer.DataCase
alias Explorer.Block
alias Explorer.Repo
alias Explorer.LatestBlock
describe "fetch/0" do
test "the latest block is copied over from the blockchain" do
use_cassette "latest_block_fetch" do
LatestBlock.fetch()
last_block = Block
|> order_by(desc: :inserted_at)
|> limit(1)
|> Repo.all
|> List.first
assert(last_block.number)
end
end
end
describe "get_latest_block/0" do
test "returns the number of the latest block" do
use_cassette "fetcher_get_latest_block" do
assert LatestBlock.get_latest_block() == "0x89923"
end
end
end
end

@ -1,53 +1,76 @@
defmodule Explorer.SkippedBlocksTest do
use Explorer.DataCase
alias Explorer.Block
alias Explorer.Repo
alias Explorer.SkippedBlocks
describe "fetch/0" do
test "inserts a missing block into the database" do
insert(:block, %{number: 2})
use_cassette "skipped_block_fetch" do
SkippedBlocks.fetch()
blocks = Block |> order_by(asc: :number) |> Repo.all |> Enum.map(fn(block) -> block.number end)
describe "first/0 when there are no blocks" do
test "returns no blocks" do
assert SkippedBlocks.first() == []
end
end
assert blocks == [1, 2]
describe "first/0 when there are no skipped blocks" do
test "returns no blocks" do
insert(:block, %{number: 0})
assert SkippedBlocks.first() == []
end
end
describe "first/0 when a block has been skipped" do
test "returns the first skipped block number" do
insert(:block, %{number: 0})
insert(:block, %{number: 2})
assert SkippedBlocks.first() == ["1"]
end
end
describe "get_skipped_blocks/0 when there are no blocks" do
describe "first/1 when there are no blocks" do
test "returns no blocks" do
assert SkippedBlocks.get_skipped_blocks() == []
assert SkippedBlocks.first(1) == []
end
end
describe "get_skipped_blocks/0 when there are no skipped blocks" do
describe "first/1 when there are no skipped blocks" do
test "returns no blocks" do
insert(:block, %{number: 0})
assert SkippedBlocks.first(1) == []
end
end
describe "first/1 when a block has been skipped" do
test "returns the skipped block number" do
insert(:block, %{number: 1})
assert SkippedBlocks.get_skipped_blocks() == []
assert SkippedBlocks.first(1) == ["0"]
end
test "returns up to the requested number of skipped block numbers in reverse order" do
insert(:block, %{number: 1})
insert(:block, %{number: 3})
assert SkippedBlocks.first(1) == ["2"]
end
describe "get_skipped_blocks/0 when a block has been skipped" do
test "returns no blocks" do
insert(:block, %{number: 2})
assert SkippedBlocks.get_skipped_blocks() == [1]
test "returns only the skipped block number" do
insert(:block, %{number: 1})
assert SkippedBlocks.first(100) == ["0"]
end
test "returns all the skipped block numbers in descending order" do
insert(:block, %{number: 1})
insert(:block, %{number: 3})
assert SkippedBlocks.first(100) == ["2", "0"]
end
end
describe "get_last_block_number/0 when there are no blocks" do
test "returns zero" do
assert SkippedBlocks.get_last_block_number() == 0
describe "latest_block_number/0 when there are no blocks" do
test "returns -1" do
assert SkippedBlocks.latest_block_number() == -1
end
end
describe "get_last_block_number/0 when there is a block" do
describe "latest_block_number/0 when there is a block" do
test "returns the number of the block" do
insert(:block, %{number: 1})
assert SkippedBlocks.get_last_block_number() == 1
assert SkippedBlocks.latest_block_number() == 1
end
end
end

@ -0,0 +1,38 @@
defmodule Explorer.Workers.ImportBlockTest do
use Explorer.DataCase
alias Explorer.Block
alias Explorer.Repo
test "perform/1 imports the requested block number as an integer" do
use_cassette "import_block_perform_1_integer" do
Explorer.Workers.ImportBlock.perform(1)
last_block = Block |> order_by(asc: :number) |> Repo.one
assert last_block.number == 1
end
end
test "perform/1 imports the requested block number as a string" do
use_cassette "import_block_perform_1_string" do
Explorer.Workers.ImportBlock.perform("1")
last_block = Block |> order_by(asc: :number) |> Repo.one
assert last_block.number == 1
end
end
test "perform/1 imports the earliest block" do
use_cassette "import_block_perform_1_earliest" do
Explorer.Workers.ImportBlock.perform("earliest")
last_block = Block |> order_by(asc: :number) |> Repo.one
assert last_block.number == 0
end
end
test "perform/1 when there is alaready a block with the requested hash" do
use_cassette "import_block_perform_1_duplicate" do
insert(:block, hash: "0x52c867bc0a91e573dc39300143c3bead7408d09d45bdb686749f02684ece72f3")
Explorer.Workers.ImportBlock.perform("1")
block_count = Block |> Repo.all |> Enum.count
assert block_count == 1
end
end
end

@ -0,0 +1,23 @@
defmodule Explorer.Workers.ImportSkippedBlocksTest do
alias Explorer.Block
alias Explorer.Repo
alias Explorer.Workers.ImportBlock
alias Explorer.Workers.ImportSkippedBlocks
import Mock
use Explorer.DataCase
describe "perform/1" do
test "imports the requested number of skipped blocks" do
insert(:block, %{number: 2})
use_cassette "import_skipped_blocks_perform_1" do
with_mock ImportBlock, [perform_later: fn (number) -> insert(:block, number: number) end] do
ImportSkippedBlocks.perform(1)
last_block = Block |> order_by(asc: :number) |> limit(1) |> Repo.one
assert last_block.number == 1
end
end
end
end
end

@ -0,0 +1,11 @@
defmodule ExplorerWeb.ExqTest do
use ExplorerWeb.FeatureCase, async: true
import Wallaby.Query, only: [css: 2]
test "views the exq dashboard", %{session: session} do
session
|> visit("/exq")
|> assert_has(css(".navbar-brand", text: "Exq"))
end
end

@ -1,21 +0,0 @@
defmodule Scrape.Backfill do
use Explorer.DataCase
alias Explorer.Block
alias Explorer.Repo
test "backfills previous blocks" do
insert(:block, %{number: 2})
use_cassette "backfill" do
Mix.Tasks.Backfill.run([])
last_block = Block
|> order_by(asc: :number)
|> limit(1)
|> Repo.all
|> List.first
assert last_block.number == 1
end
end
end

@ -1,19 +0,0 @@
defmodule Scrape.Test do
use Explorer.DataCase
alias Explorer.Block
alias Explorer.Repo
test "it downloads a new block" do
use_cassette "scrape" do
Mix.Tasks.Scrape.run([])
last_block = Block
|> order_by(desc: :inserted_at)
|> limit(1)
|> Repo.all
|> List.first
assert(last_block.number)
end
end
end

@ -0,0 +1,30 @@
[
{
"request": {
"body": "{\"params\":[\"1\",true],\"method\":\"eth_getBlockByNumber\",\"jsonrpc\":\"2.0\",\"id\":3}",
"headers": {
"Content-Type": "application/json"
},
"method": "post",
"options": [],
"request_body": "",
"url": "https://sokol.poa.network:443"
},
"response": {
"binary": false,
"body": "{\"jsonrpc\":\"2.0\",\"result\":{\"author\":\"0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca\",\"difficulty\":\"0xffffffffffffffffffffffffedf58e45\",\"extraData\":\"0xd5830108048650617269747986312e32322e31826c69\",\"gasLimit\":\"0x66556d\",\"gasUsed\":\"0x0\",\"hash\":\"0x52c867bc0a91e573dc39300143c3bead7408d09d45bdb686749f02684ece72f3\",\"logsBloom\":\"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000\",\"miner\":\"0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca\",\"number\":\"0x1\",\"parentHash\":\"0x5b28c1bfd3a15230c9a46b399cd0f9a6920d432e85381cc6a140b06e8410112f\",\"receiptsRoot\":\"0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421\",\"sealFields\":[\"0x84120a71ba\",\"0xb8417a5887662f09ac4673af5850d28f3ad6550407b9c814ef563a13320f881b55ef03754f48f2dde027ad4a5abcabcc42780d9ebfc645f183e5252507d6a25bc2ec01\"],\"sha3Uncles\":\"0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347\",\"signature\":\"7a5887662f09ac4673af5850d28f3ad6550407b9c814ef563a13320f881b55ef03754f48f2dde027ad4a5abcabcc42780d9ebfc645f183e5252507d6a25bc2ec01\",\"size\":\"0x240\",\"stateRoot\":\"0xc196ad59d867542ef20b29df5f418d07dc7234f4bc3d25260526620b7958a8fb\",\"step\":\"302674362\",\"timestamp\":\"0x5a3438a2\",\"totalDifficulty\":\"0xffffffffffffffffffffffffedf78e45\",\"transactions\":[],\"transactionsRoot\":\"0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421\",\"uncles\":[]},\"id\":3}\n",
"headers": {
"Date": "Sun, 04 Feb 2018 07:41:06 GMT",
"Content-Type": "application/json",
"Transfer-Encoding": "chunked",
"Connection": "keep-alive",
"Set-Cookie": "__cfduid=d9be038a0189dadf6dd2c0f72a4626d271517730066; expires=Mon, 04-Feb-19 07:41:06 GMT; path=/; domain=.poa.network; HttpOnly; Secure",
"Expect-CT": "max-age=604800, report-uri=\"https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct\"",
"Server": "cloudflare",
"CF-RAY": "3e7bfc51196092e8-SJC"
},
"status_code": 200,
"type": "ok"
}
}
]

@ -0,0 +1,30 @@
[
{
"request": {
"body": "{\"params\":[\"earliest\",true],\"method\":\"eth_getBlockByNumber\",\"jsonrpc\":\"2.0\",\"id\":2}",
"headers": {
"Content-Type": "application/json"
},
"method": "post",
"options": [],
"request_body": "",
"url": "https://sokol.poa.network:443"
},
"response": {
"binary": false,
"body": "{\"jsonrpc\":\"2.0\",\"result\":{\"author\":\"0x0000000000000000000000000000000000000000\",\"difficulty\":\"0x20000\",\"extraData\":\"0x\",\"gasLimit\":\"0x663be0\",\"gasUsed\":\"0x0\",\"hash\":\"0x5b28c1bfd3a15230c9a46b399cd0f9a6920d432e85381cc6a140b06e8410112f\",\"logsBloom\":\"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000\",\"miner\":\"0x0000000000000000000000000000000000000000\",\"number\":\"0x0\",\"parentHash\":\"0x0000000000000000000000000000000000000000000000000000000000000000\",\"receiptsRoot\":\"0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421\",\"sealFields\":[\"0x80\",\"0xb8410000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000\"],\"sha3Uncles\":\"0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347\",\"signature\":\"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000\",\"size\":\"0x215\",\"stateRoot\":\"0xfad4af258fd11939fae0c6c6eec9d340b1caac0b0196fd9a1bc3f489c5bf00b3\",\"step\":\"0\",\"timestamp\":\"0x0\",\"totalDifficulty\":\"0x20000\",\"transactions\":[],\"transactionsRoot\":\"0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421\",\"uncles\":[]},\"id\":2}\n",
"headers": {
"Date": "Sun, 04 Feb 2018 01:09:56 GMT",
"Content-Type": "application/json",
"Transfer-Encoding": "chunked",
"Connection": "keep-alive",
"Set-Cookie": "__cfduid=ddc1b2cd399b32475cbc191f806ff29d61517706596; expires=Mon, 04-Feb-19 01:09:56 GMT; path=/; domain=.poa.network; HttpOnly; Secure",
"Expect-CT": "max-age=604800, report-uri=\"https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct\"",
"Server": "cloudflare",
"CF-RAY": "3e79bf530a669601-SJC"
},
"status_code": 200,
"type": "ok"
}
}
]

@ -0,0 +1,30 @@
[
{
"request": {
"body": "{\"params\":[\"1\",true],\"method\":\"eth_getBlockByNumber\",\"jsonrpc\":\"2.0\",\"id\":0}",
"headers": {
"Content-Type": "application/json"
},
"method": "post",
"options": [],
"request_body": "",
"url": "https://sokol.poa.network:443"
},
"response": {
"binary": false,
"body": "{\"jsonrpc\":\"2.0\",\"result\":{\"author\":\"0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca\",\"difficulty\":\"0xffffffffffffffffffffffffedf58e45\",\"extraData\":\"0xd5830108048650617269747986312e32322e31826c69\",\"gasLimit\":\"0x66556d\",\"gasUsed\":\"0x0\",\"hash\":\"0x52c867bc0a91e573dc39300143c3bead7408d09d45bdb686749f02684ece72f3\",\"logsBloom\":\"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000\",\"miner\":\"0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca\",\"number\":\"0x1\",\"parentHash\":\"0x5b28c1bfd3a15230c9a46b399cd0f9a6920d432e85381cc6a140b06e8410112f\",\"receiptsRoot\":\"0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421\",\"sealFields\":[\"0x84120a71ba\",\"0xb8417a5887662f09ac4673af5850d28f3ad6550407b9c814ef563a13320f881b55ef03754f48f2dde027ad4a5abcabcc42780d9ebfc645f183e5252507d6a25bc2ec01\"],\"sha3Uncles\":\"0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347\",\"signature\":\"7a5887662f09ac4673af5850d28f3ad6550407b9c814ef563a13320f881b55ef03754f48f2dde027ad4a5abcabcc42780d9ebfc645f183e5252507d6a25bc2ec01\",\"size\":\"0x240\",\"stateRoot\":\"0xc196ad59d867542ef20b29df5f418d07dc7234f4bc3d25260526620b7958a8fb\",\"step\":\"302674362\",\"timestamp\":\"0x5a3438a2\",\"totalDifficulty\":\"0xffffffffffffffffffffffffedf78e45\",\"transactions\":[],\"transactionsRoot\":\"0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421\",\"uncles\":[]},\"id\":0}\n",
"headers": {
"Date": "Sun, 04 Feb 2018 01:09:55 GMT",
"Content-Type": "application/json",
"Transfer-Encoding": "chunked",
"Connection": "keep-alive",
"Set-Cookie": "__cfduid=dd200d77362e268d446c7df8154111eac1517706595; expires=Mon, 04-Feb-19 01:09:55 GMT; path=/; domain=.poa.network; HttpOnly; Secure",
"Expect-CT": "max-age=604800, report-uri=\"https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct\"",
"Server": "cloudflare",
"CF-RAY": "3e79bf4df8849601-SJC"
},
"status_code": 200,
"type": "ok"
}
}
]

@ -0,0 +1,30 @@
[
{
"request": {
"body": "{\"params\":[\"1\",true],\"method\":\"eth_getBlockByNumber\",\"jsonrpc\":\"2.0\",\"id\":1}",
"headers": {
"Content-Type": "application/json"
},
"method": "post",
"options": [],
"request_body": "",
"url": "https://sokol.poa.network:443"
},
"response": {
"binary": false,
"body": "{\"jsonrpc\":\"2.0\",\"result\":{\"author\":\"0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca\",\"difficulty\":\"0xffffffffffffffffffffffffedf58e45\",\"extraData\":\"0xd5830108048650617269747986312e32322e31826c69\",\"gasLimit\":\"0x66556d\",\"gasUsed\":\"0x0\",\"hash\":\"0x52c867bc0a91e573dc39300143c3bead7408d09d45bdb686749f02684ece72f3\",\"logsBloom\":\"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000\",\"miner\":\"0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca\",\"number\":\"0x1\",\"parentHash\":\"0x5b28c1bfd3a15230c9a46b399cd0f9a6920d432e85381cc6a140b06e8410112f\",\"receiptsRoot\":\"0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421\",\"sealFields\":[\"0x84120a71ba\",\"0xb8417a5887662f09ac4673af5850d28f3ad6550407b9c814ef563a13320f881b55ef03754f48f2dde027ad4a5abcabcc42780d9ebfc645f183e5252507d6a25bc2ec01\"],\"sha3Uncles\":\"0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347\",\"signature\":\"7a5887662f09ac4673af5850d28f3ad6550407b9c814ef563a13320f881b55ef03754f48f2dde027ad4a5abcabcc42780d9ebfc645f183e5252507d6a25bc2ec01\",\"size\":\"0x240\",\"stateRoot\":\"0xc196ad59d867542ef20b29df5f418d07dc7234f4bc3d25260526620b7958a8fb\",\"step\":\"302674362\",\"timestamp\":\"0x5a3438a2\",\"totalDifficulty\":\"0xffffffffffffffffffffffffedf78e45\",\"transactions\":[],\"transactionsRoot\":\"0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421\",\"uncles\":[]},\"id\":1}\n",
"headers": {
"Date": "Sun, 04 Feb 2018 01:09:56 GMT",
"Content-Type": "application/json",
"Transfer-Encoding": "chunked",
"Connection": "keep-alive",
"Set-Cookie": "__cfduid=dd200d77362e268d446c7df8154111eac1517706595; expires=Mon, 04-Feb-19 01:09:55 GMT; path=/; domain=.poa.network; HttpOnly; Secure",
"Expect-CT": "max-age=604800, report-uri=\"https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct\"",
"Server": "cloudflare",
"CF-RAY": "3e79bf4fa9299601-SJC"
},
"status_code": 200,
"type": "ok"
}
}
]
Loading…
Cancel
Save