Extract Explorer.Indexer to Indexer

Resolves #304
pull/331/head
Luke Imhoff 7 years ago
parent 1c0bfe5eb0
commit 6b3fd80916
  1. 6
      apps/ethereum_jsonrpc/mix.exs
  2. 3
      apps/ethereum_jsonrpc/test/test_helper.exs
  3. 4
      apps/explorer/config/config.exs
  4. 1
      apps/explorer/lib/explorer/application.ex
  5. 4
      apps/explorer/mix.exs
  6. 3
      apps/explorer/test/test_helper.exs
  7. 8
      apps/explorer_web/mix.exs
  8. 7
      apps/explorer_web/test/test_helper.exs
  9. 29
      apps/indexer/config/config.exs
  10. 22
      apps/indexer/lib/indexer.ex
  11. 8
      apps/indexer/lib/indexer/address_balance_fetcher.ex
  12. 14
      apps/indexer/lib/indexer/address_extraction.ex
  13. 22
      apps/indexer/lib/indexer/application.ex
  14. 14
      apps/indexer/lib/indexer/block_fetcher.ex
  15. 4
      apps/indexer/lib/indexer/buffered_task.ex
  16. 8
      apps/indexer/lib/indexer/internal_transaction_fetcher.ex
  17. 10
      apps/indexer/lib/indexer/pending_transaction_fetcher.ex
  18. 2
      apps/indexer/lib/indexer/sequence.ex
  19. 21
      apps/indexer/mix.exs
  20. 6
      apps/indexer/test/indexer/address_balance_fetcher_test.exs
  21. 16
      apps/indexer/test/indexer/address_extraction_test.exs
  22. 16
      apps/indexer/test/indexer/block_fetcher_test.exs
  23. 4
      apps/indexer/test/indexer/buffered_task_test.exs
  24. 12
      apps/indexer/test/indexer/internal_transaction_fetcher_test.exs
  25. 6
      apps/indexer/test/indexer/pending_transaction_fetcher_test.exs
  26. 4
      apps/indexer/test/indexer/sequence_test.exs
  27. 4
      apps/indexer/test/indexer_test.exs
  28. 4
      apps/indexer/test/support/indexer/address_balance_fetcher_case.ex
  29. 4
      apps/indexer/test/support/indexer/internal_transaction_fetcher_case.ex
  30. 19
      apps/indexer/test/test_helper.exs
  31. 2
      coveralls.json
  32. 12
      mix.exs

@ -38,7 +38,11 @@ defmodule EthereumJsonrpc.MixProject do
end end
defp aliases(env) do defp aliases(env) do
env_aliases(env) [
# to match behavior of `mix test` from project root, which needs to not start applications for `indexer` to
# prevent its supervision tree from starting, which is undesirable in test
test: "test --no-start"
] ++ env_aliases(env)
end end
defp env_aliases(:dev), do: [] defp env_aliases(:dev), do: []

@ -3,5 +3,8 @@ junit_folder = Mix.Project.build_path() <> "/junit/#{Mix.Project.config()[:app]}
File.mkdir_p!(junit_folder) File.mkdir_p!(junit_folder)
:ok = Application.put_env(:junit_formatter, :report_dir, junit_folder) :ok = Application.put_env(:junit_formatter, :report_dir, junit_folder)
# Counter `test --no-start`. `--no-start` is needed for `:indexer` compatibility
{:ok, _} = Application.ensure_all_started(:ethereum_jsonrpc)
ExUnit.configure(formatters: [JUnitFormatter, ExUnit.CLIFormatter]) ExUnit.configure(formatters: [JUnitFormatter, ExUnit.CLIFormatter])
ExUnit.start() ExUnit.start()

@ -7,10 +7,6 @@ use Mix.Config
config :ecto, json_library: Jason config :ecto, json_library: Jason
config :explorer, :indexer,
block_rate: 5_000,
debug_logs: !!System.get_env("DEBUG_INDEXER")
# General application configuration # General application configuration
config :explorer, config :explorer,
ecto_repos: [Explorer.Repo], ecto_repos: [Explorer.Repo],

@ -25,7 +25,6 @@ defmodule Explorer.Application do
[ [
configure(Explorer.Chain.Statistics.Server), configure(Explorer.Chain.Statistics.Server),
configure(Explorer.ExchangeRates), configure(Explorer.ExchangeRates),
configure(Explorer.Indexer.Supervisor),
configure(Explorer.Market.History.Cataloger) configure(Explorer.Market.History.Cataloger)
] ]
|> List.flatten() |> List.flatten()

@ -75,8 +75,6 @@ defmodule Explorer.Mixfile do
# Code coverage # Code coverage
{:excoveralls, "~> 0.8.1", only: [:test]}, {:excoveralls, "~> 0.8.1", only: [:test]},
{:exvcr, "~> 0.10", only: :test}, {:exvcr, "~> 0.10", only: :test},
# JSONRPC access to Parity for `Explorer.Indexer`
{:ethereum_jsonrpc, in_umbrella: true},
{:httpoison, "~> 1.0", override: true}, {:httpoison, "~> 1.0", override: true},
{:jason, "~> 1.0"}, {:jason, "~> 1.0"},
{:junit_formatter, ">= 0.0.0", only: [:test], runtime: false}, {:junit_formatter, ">= 0.0.0", only: [:test], runtime: false},
@ -100,7 +98,7 @@ defmodule Explorer.Mixfile do
[ [
"ecto.setup": ["ecto.create", "ecto.migrate", "run priv/repo/seeds.exs"], "ecto.setup": ["ecto.create", "ecto.migrate", "run priv/repo/seeds.exs"],
"ecto.reset": ["ecto.drop", "ecto.setup"], "ecto.reset": ["ecto.drop", "ecto.setup"],
test: ["ecto.create --quiet", "ecto.migrate", "test"] test: ["ecto.create --quiet", "ecto.migrate", "test --no-start"]
] ++ env_aliases(env) ] ++ env_aliases(env)
end end

@ -3,6 +3,9 @@ junit_folder = Mix.Project.build_path() <> "/junit/#{Mix.Project.config()[:app]}
File.mkdir_p!(junit_folder) File.mkdir_p!(junit_folder)
:ok = Application.put_env(:junit_formatter, :report_dir, junit_folder) :ok = Application.put_env(:junit_formatter, :report_dir, junit_folder)
# Counter `test --no-start`. `--no-start` is needed for `:indexer` compatibility
{:ok, _} = Application.ensure_all_started(:explorer)
ExUnit.configure(formatters: [JUnitFormatter, ExUnit.CLIFormatter]) ExUnit.configure(formatters: [JUnitFormatter, ExUnit.CLIFormatter])
ExUnit.start() ExUnit.start()

@ -112,7 +112,13 @@ defmodule ExplorerWeb.Mixfile do
compile: "compile --warnings-as-errors", compile: "compile --warnings-as-errors",
"ecto.setup": ["ecto.create", "ecto.migrate", "run priv/repo/seeds.exs"], "ecto.setup": ["ecto.create", "ecto.migrate", "run priv/repo/seeds.exs"],
"ecto.reset": ["ecto.drop", "ecto.setup"], "ecto.reset": ["ecto.drop", "ecto.setup"],
test: ["ecto.create --quiet", "ecto.migrate", "test"] test: [
"ecto.create --quiet",
"ecto.migrate",
# to match behavior of `mix test` from project root, which needs to not start applications for `indexer` to
# prevent its supervision tree from starting, which is undesirable in test
"test --no-start"
]
] ]
end end

@ -3,12 +3,15 @@ junit_folder = Mix.Project.build_path() <> "/junit/#{Mix.Project.config()[:app]}
File.mkdir_p!(junit_folder) File.mkdir_p!(junit_folder)
:ok = Application.put_env(:junit_formatter, :report_dir, junit_folder) :ok = Application.put_env(:junit_formatter, :report_dir, junit_folder)
ExUnit.configure(formatters: [JUnitFormatter, ExUnit.CLIFormatter]) # Counter `test --no-start`. `--no-start` is needed for `:indexer` compatibility
ExUnit.start() {:ok, _} = Application.ensure_all_started(:explorer_web)
{:ok, _} = Application.ensure_all_started(:wallaby) {:ok, _} = Application.ensure_all_started(:wallaby)
Application.put_env(:wallaby, :base_url, ExplorerWeb.Endpoint.url()) Application.put_env(:wallaby, :base_url, ExplorerWeb.Endpoint.url())
{:ok, _} = Application.ensure_all_started(:ex_machina) {:ok, _} = Application.ensure_all_started(:ex_machina)
ExUnit.configure(formatters: [JUnitFormatter, ExUnit.CLIFormatter])
ExUnit.start()
Ecto.Adapters.SQL.Sandbox.mode(Explorer.Repo, :manual) Ecto.Adapters.SQL.Sandbox.mode(Explorer.Repo, :manual)

@ -2,29 +2,8 @@
# and its dependencies with the aid of the Mix.Config module. # and its dependencies with the aid of the Mix.Config module.
use Mix.Config use Mix.Config
# This configuration is loaded before any dependency and is restricted config :indexer,
# to this project. If another project depends on this project, this block_rate: 5_000,
# file won't be loaded nor affect the parent project. For this reason, debug_logs: !!System.get_env("DEBUG_INDEXER")
# if you want to provide default values for your application for
# 3rd-party users, it should be done in your "mix.exs" file.
# You can configure your application as: config :indexer, ecto_repos: [Explorer.Repo]
#
# config :indexer, key: :value
#
# and access this configuration in your application as:
#
# Application.get_env(:indexer, :key)
#
# You can also configure a 3rd-party app:
#
# config :logger, level: :info
#
# It is also possible to import configuration files, relative to this
# directory. For example, you can emulate configuration per environment
# by uncommenting the line below and defining dev.exs, test.exs and such.
# Configuration from the imported file will override the ones defined
# here (which is why it is important to import them last).
#
# import_config "#{Mix.env}.exs"

@ -1,10 +1,10 @@
defmodule Explorer.Indexer do defmodule Indexer do
@moduledoc """ @moduledoc """
Indexes an Ethereum-based chain using JSONRPC. Indexes an Ethereum-based chain using JSONRPC.
""" """
require Logger require Logger
alias Explorer.{Chain, Indexer} alias Explorer.Chain
@doc """ @doc """
The maximum `t:Explorer.Chain.Block.t/0` `number` that was indexed The maximum `t:Explorer.Chain.Block.t/0` `number` that was indexed
@ -13,12 +13,12 @@ defmodule Explorer.Indexer do
iex> insert(:block, number: 2) iex> insert(:block, number: 2)
iex> insert(:block, number: 1) iex> insert(:block, number: 1)
iex> Explorer.Indexer.max_block_number() iex> Indexer.max_block_number()
2 2
If there are no blocks, `0` is returned to indicate to index from genesis block. If there are no blocks, `0` is returned to indicate to index from genesis block.
iex> Explorer.Indexer.max_block_number() iex> Indexer.max_block_number()
0 0
""" """
@ -34,16 +34,16 @@ defmodule Explorer.Indexer do
When there are no blocks the next block is the 0th block When there are no blocks the next block is the 0th block
iex> Explorer.Indexer.max_block_number() iex> Indexer.max_block_number()
0 0
iex> Explorer.Indexer.next_block_number() iex> Indexer.next_block_number()
0 0
When there is a block, it is the successive block number When there is a block, it is the successive block number
iex> insert(:block, number: 2) iex> insert(:block, number: 2)
iex> insert(:block, number: 1) iex> insert(:block, number: 1)
iex> Explorer.Indexer.next_block_number() iex> Indexer.next_block_number()
3 3
""" """
@ -69,14 +69,14 @@ defmodule Explorer.Indexer do
Enables debug logs for indexing system. Enables debug logs for indexing system.
""" """
def enable_debug_logs do def enable_debug_logs do
Application.put_env(:explorer, :indexer, Keyword.put(config(), :debug_logs, true)) Application.put_env(:indexer, :debug_logs, true)
end end
@doc """ @doc """
Disables debug logs for indexing system. Disables debug logs for indexing system.
""" """
def disable_debug_logs do def disable_debug_logs do
Application.put_env(:explorer, :indexer, Keyword.put(config(), :debug_logs, false)) Application.put_env(:indexer, :debug_logs, false)
end end
@doc """ @doc """
@ -90,8 +90,6 @@ defmodule Explorer.Indexer do
end end
defp debug_logs_enabled? do defp debug_logs_enabled? do
Keyword.fetch!(config(), :debug_logs) Application.fetch_env!(:indexer, :debug_logs)
end end
defp config, do: Application.fetch_env!(:explorer, :indexer)
end end

@ -1,13 +1,13 @@
defmodule Explorer.Indexer.AddressBalanceFetcher do defmodule Indexer.AddressBalanceFetcher do
@moduledoc """ @moduledoc """
Fetches `t:Explorer.Chain.Address.t/0` `fetched_balance`. Fetches `t:Explorer.Chain.Address.t/0` `fetched_balance`.
""" """
import EthereumJSONRPC, only: [integer_to_quantity: 1] import EthereumJSONRPC, only: [integer_to_quantity: 1]
alias Explorer.{BufferedTask, Chain} alias Explorer.Chain
alias Explorer.Chain.{Block, Hash} alias Explorer.Chain.{Block, Hash}
alias Explorer.Indexer alias Indexer.BufferedTask
@behaviour BufferedTask @behaviour BufferedTask
@ -16,7 +16,7 @@ defmodule Explorer.Indexer.AddressBalanceFetcher do
max_batch_size: 500, max_batch_size: 500,
max_concurrency: 4, max_concurrency: 4,
init_chunk_size: 1000, init_chunk_size: 1000,
task_supervisor: Explorer.Indexer.TaskSupervisor task_supervisor: Indexer.TaskSupervisor
] ]
@doc """ @doc """

@ -1,4 +1,4 @@
defmodule Explorer.Indexer.AddressExtraction do defmodule Indexer.AddressExtraction do
@moduledoc """ @moduledoc """
Extract Addresses from data fetched from the Blockchain and structured as Blocks, InternalTransactions, Extract Addresses from data fetched from the Blockchain and structured as Blocks, InternalTransactions,
Transactions and Logs. Transactions and Logs.
@ -104,7 +104,7 @@ defmodule Explorer.Indexer.AddressExtraction do
Blocks have their `miner_hash` extracted. Blocks have their `miner_hash` extracted.
iex> Explorer.Indexer.AddressExtraction.extract_addresses( iex> Indexer.AddressExtraction.extract_addresses(
...> %{ ...> %{
...> blocks: [ ...> blocks: [
...> %{ ...> %{
@ -124,7 +124,7 @@ defmodule Explorer.Indexer.AddressExtraction do
Internal transactions can have their `from_address_hash`, `to_address_hash` and/or `created_contract_address_hash` Internal transactions can have their `from_address_hash`, `to_address_hash` and/or `created_contract_address_hash`
extracted. extracted.
iex> Explorer.Indexer.AddressExtraction.extract_addresses( iex> Indexer.AddressExtraction.extract_addresses(
...> %{ ...> %{
...> internal_transactions: [ ...> internal_transactions: [
...> %{ ...> %{
@ -161,7 +161,7 @@ defmodule Explorer.Indexer.AddressExtraction do
Transactions can have their `from_address_hash` and/or `to_address_hash` extracted. Transactions can have their `from_address_hash` and/or `to_address_hash` extracted.
iex> Explorer.Indexer.AddressExtraction.extract_addresses( iex> Indexer.AddressExtraction.extract_addresses(
...> %{ ...> %{
...> transactions: [ ...> transactions: [
...> %{ ...> %{
@ -193,7 +193,7 @@ defmodule Explorer.Indexer.AddressExtraction do
Logs can have their `address_hash` extracted. Logs can have their `address_hash` extracted.
iex> Explorer.Indexer.AddressExtraction.extract_addresses( iex> Indexer.AddressExtraction.extract_addresses(
...> %{ ...> %{
...> logs: [ ...> logs: [
...> %{ ...> %{
@ -212,7 +212,7 @@ defmodule Explorer.Indexer.AddressExtraction do
When the same address is mentioned multiple times, the greatest `block_number` is used When the same address is mentioned multiple times, the greatest `block_number` is used
iex> Explorer.Indexer.AddressExtraction.extract_addresses( iex> Indexer.AddressExtraction.extract_addresses(
...> %{ ...> %{
...> blocks: [ ...> blocks: [
...> %{ ...> %{
@ -262,7 +262,7 @@ defmodule Explorer.Indexer.AddressExtraction do
When a contract is created and then used in internal transactions and transaction in the same fetched data, the When a contract is created and then used in internal transactions and transaction in the same fetched data, the
`created_contract_code` is merged with the greatest `block_number` `created_contract_code` is merged with the greatest `block_number`
iex> Explorer.Indexer.AddressExtraction.extract_addresses( iex> Indexer.AddressExtraction.extract_addresses(
...> %{ ...> %{
...> internal_transactions: [ ...> internal_transactions: [
...> %{ ...> %{

@ -1,26 +1,24 @@
defmodule Explorer.Indexer.Supervisor do defmodule Indexer.Application do
@moduledoc """ @moduledoc """
Supervising the fetchers for the `Explorer.Indexer` This is the `Application` module for `Indexer`.
""" """
use Supervisor use Application
alias Explorer.Indexer.{AddressBalanceFetcher, BlockFetcher, InternalTransactionFetcher, PendingTransactionFetcher} alias Indexer.{AddressBalanceFetcher, BlockFetcher, InternalTransactionFetcher, PendingTransactionFetcher}
def start_link(opts) do @impl Application
Supervisor.start_link(__MODULE__, opts, name: __MODULE__) def start(_type, _args) do
end
@impl Supervisor
def init(_opts) do
children = [ children = [
{Task.Supervisor, name: Explorer.Indexer.TaskSupervisor}, {Task.Supervisor, name: Indexer.TaskSupervisor},
{AddressBalanceFetcher, name: AddressBalanceFetcher}, {AddressBalanceFetcher, name: AddressBalanceFetcher},
{PendingTransactionFetcher, name: PendingTransactionFetcher}, {PendingTransactionFetcher, name: PendingTransactionFetcher},
{InternalTransactionFetcher, name: InternalTransactionFetcher}, {InternalTransactionFetcher, name: InternalTransactionFetcher},
{BlockFetcher, []} {BlockFetcher, []}
] ]
Supervisor.init(children, strategy: :one_for_one) opts = [strategy: :one_for_one, name: Indexer.Supervisor]
Supervisor.start_link(children, opts)
end end
end end

@ -1,4 +1,4 @@
defmodule Explorer.Indexer.BlockFetcher do defmodule Indexer.BlockFetcher do
@moduledoc """ @moduledoc """
Fetches and indexes block ranges from gensis to realtime. Fetches and indexes block ranges from gensis to realtime.
""" """
@ -7,12 +7,12 @@ defmodule Explorer.Indexer.BlockFetcher do
require Logger require Logger
import Explorer.Indexer, only: [debug: 1] import Indexer, only: [debug: 1]
alias EthereumJSONRPC alias EthereumJSONRPC
alias EthereumJSONRPC.Transactions alias EthereumJSONRPC.Transactions
alias Explorer.{BufferedTask, Chain, Indexer} alias Explorer.Chain
alias Explorer.Indexer.{AddressBalanceFetcher, AddressExtraction, InternalTransactionFetcher, Sequence} alias Indexer.{AddressBalanceFetcher, AddressExtraction, BufferedTask, InternalTransactionFetcher, Sequence}
# dialyzer thinks that Logger.debug functions always have no_local_return # dialyzer thinks that Logger.debug functions always have no_local_return
@dialyzer {:nowarn_function, import_range: 3} @dialyzer {:nowarn_function, import_range: 3}
@ -63,7 +63,11 @@ defmodule Explorer.Indexer.BlockFetcher do
@impl GenServer @impl GenServer
def init(opts) do def init(opts) do
opts = Keyword.merge(Application.fetch_env!(:explorer, :indexer), opts) opts =
:indexer
|> Application.get_all_env()
|> Keyword.merge(opts)
:timer.send_interval(15_000, self(), :debug_count) :timer.send_interval(15_000, self(), :debug_count)
state = %{ state = %{

@ -1,4 +1,4 @@
defmodule Explorer.BufferedTask do defmodule Indexer.BufferedTask do
@moduledoc """ @moduledoc """
Provides a behaviour for batched task running with retries. Provides a behaviour for batched task running with retries.
@ -166,7 +166,7 @@ defmodule Explorer.BufferedTask do
]} ]}
) :: {:ok, pid()} | {:error, {:already_started, pid()}} ) :: {:ok, pid()} | {:error, {:already_started, pid()}}
def start_link({module, base_opts}) do def start_link({module, base_opts}) do
default_opts = Application.fetch_env!(:explorer, :indexer) default_opts = Application.get_all_env(:indexer)
opts = Keyword.merge(default_opts, base_opts) opts = Keyword.merge(default_opts, base_opts)
GenServer.start_link(__MODULE__, {module, opts}, name: opts[:name]) GenServer.start_link(__MODULE__, {module, opts}, name: opts[:name])

@ -1,4 +1,4 @@
defmodule Explorer.Indexer.InternalTransactionFetcher do defmodule Indexer.InternalTransactionFetcher do
@moduledoc """ @moduledoc """
Fetches and indexes `t:Explorer.Chain.InternalTransaction.t/0`. Fetches and indexes `t:Explorer.Chain.InternalTransaction.t/0`.
@ -7,8 +7,8 @@ defmodule Explorer.Indexer.InternalTransactionFetcher do
require Logger require Logger
alias Explorer.{BufferedTask, Chain, Indexer} alias Explorer.Chain
alias Explorer.Indexer.{AddressBalanceFetcher, AddressExtraction} alias Indexer.{AddressBalanceFetcher, AddressExtraction, BufferedTask}
alias Explorer.Chain.{Block, Hash} alias Explorer.Chain.{Block, Hash}
@behaviour BufferedTask @behaviour BufferedTask
@ -20,7 +20,7 @@ defmodule Explorer.Indexer.InternalTransactionFetcher do
max_concurrency: @max_concurrency, max_concurrency: @max_concurrency,
max_batch_size: @max_batch_size, max_batch_size: @max_batch_size,
init_chunk_size: 5000, init_chunk_size: 5000,
task_supervisor: Explorer.Indexer.TaskSupervisor task_supervisor: Indexer.TaskSupervisor
] ]
@doc """ @doc """

@ -1,4 +1,4 @@
defmodule Explorer.Indexer.PendingTransactionFetcher do defmodule Indexer.PendingTransactionFetcher do
@moduledoc """ @moduledoc """
Fetches pending transactions and imports them. Fetches pending transactions and imports them.
@ -11,8 +11,8 @@ defmodule Explorer.Indexer.PendingTransactionFetcher do
import EthereumJSONRPC.Parity, only: [fetch_pending_transactions: 0] import EthereumJSONRPC.Parity, only: [fetch_pending_transactions: 0]
alias Explorer.{Chain, Indexer} alias Explorer.Chain
alias Explorer.Indexer.{AddressExtraction, PendingTransactionFetcher} alias Indexer.{AddressExtraction, PendingTransactionFetcher}
# milliseconds # milliseconds
@default_interval 1_000 @default_interval 1_000
@ -46,8 +46,8 @@ defmodule Explorer.Indexer.PendingTransactionFetcher do
@impl GenServer @impl GenServer
def init(opts) do def init(opts) do
opts = opts =
:explorer :indexer
|> Application.fetch_env!(:indexer) |> Application.get_all_env()
|> Keyword.merge(opts) |> Keyword.merge(opts)
state = state =

@ -1,4 +1,4 @@
defmodule Explorer.Indexer.Sequence do defmodule Indexer.Sequence do
@moduledoc false @moduledoc false
use Agent use Agent

@ -3,6 +3,7 @@ defmodule Indexer.MixProject do
def project do def project do
[ [
aliases: aliases(),
app: :indexer, app: :indexer,
version: "0.1.0", version: "0.1.0",
build_path: "../../_build", build_path: "../../_build",
@ -10,6 +11,7 @@ defmodule Indexer.MixProject do
deps_path: "../../deps", deps_path: "../../deps",
lockfile: "../../mix.lock", lockfile: "../../mix.lock",
elixir: "~> 1.6", elixir: "~> 1.6",
elixirc_paths: elixirc_paths(Mix.env()),
start_permanent: Mix.env() == :prod, start_permanent: Mix.env() == :prod,
deps: deps() deps: deps()
] ]
@ -23,12 +25,25 @@ defmodule Indexer.MixProject do
] ]
end end
defp aliases do
[
# so that the supervision tree does not start, which would begin indexing, and so that the various fetchers can
# be started with `ExUnit`'s `start_supervised` for unit testing.
test: "test --no-start"
]
end
# Run "mix help deps" to learn about dependencies. # Run "mix help deps" to learn about dependencies.
defp deps do defp deps do
[ [
# {:dep_from_hexpm, "~> 0.3.0"}, # JSONRPC access to Parity for `Explorer.Indexer`
# {:dep_from_git, git: "https://github.com/elixir-lang/my_dep.git", tag: "0.1.0"}, {:ethereum_jsonrpc, in_umbrella: true},
# {:sibling_app_in_umbrella, in_umbrella: true}, # Importing to database
{:explorer, in_umbrella: true}
] ]
end end
# Specifies which paths to compile per environment.
defp elixirc_paths(:test), do: ["test/support" | elixirc_paths(:dev)]
defp elixirc_paths(_), do: ["lib"]
end end

@ -1,10 +1,10 @@
defmodule Explorer.Indexer.AddressBalanceFetcherTest do defmodule Indexer.AddressBalanceFetcherTest do
# MUST be `async: false` so that {:shared, pid} is set for connection to allow AddressBalanceFetcher's self-send to have # MUST be `async: false` so that {:shared, pid} is set for connection to allow AddressBalanceFetcher's self-send to have
# connection allowed immediately. # connection allowed immediately.
use Explorer.DataCase, async: false use Explorer.DataCase, async: false
alias Explorer.Chain.{Address, Hash, Wei} alias Explorer.Chain.{Address, Hash, Wei}
alias Explorer.Indexer.{AddressBalanceFetcher, AddressBalanceFetcherCase} alias Indexer.{AddressBalanceFetcher, AddressBalanceFetcherCase}
@block_number 2_932_838 @block_number 2_932_838
@hash %Explorer.Chain.Hash{ @hash %Explorer.Chain.Hash{
@ -13,7 +13,7 @@ defmodule Explorer.Indexer.AddressBalanceFetcherTest do
} }
setup do setup do
start_supervised!({Task.Supervisor, name: Explorer.Indexer.TaskSupervisor}) start_supervised!({Task.Supervisor, name: Indexer.TaskSupervisor})
:ok :ok
end end

@ -1,13 +1,13 @@
defmodule Explorer.Indexer.AddressExtractionTest do defmodule Indexer.AddressExtractionTest do
use Explorer.DataCase, async: true use Explorer.DataCase, async: true
alias Explorer.Indexer.AddressExtraction alias Indexer.AddressExtraction
doctest AddressExtraction doctest AddressExtraction
describe "extract_addresses/1" do describe "extract_addresses/1" do
test "blocks without a `miner_hash` aren't extracted" do test "blocks without a `miner_hash` aren't extracted" do
assert Explorer.Indexer.AddressExtraction.extract_addresses(%{ assert Indexer.AddressExtraction.extract_addresses(%{
blocks: [ blocks: [
%{ %{
number: 34 number: 34
@ -17,7 +17,7 @@ defmodule Explorer.Indexer.AddressExtractionTest do
end end
test "blocks without a `number` aren't extracted" do test "blocks without a `number` aren't extracted" do
assert Explorer.Indexer.AddressExtraction.extract_addresses(%{ assert Indexer.AddressExtraction.extract_addresses(%{
blocks: [ blocks: [
%{ %{
miner_hash: "0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca" miner_hash: "0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca"
@ -27,7 +27,7 @@ defmodule Explorer.Indexer.AddressExtractionTest do
end end
test "internal_transactions with a `from_address_hash` without a `block_number` aren't extracted" do test "internal_transactions with a `from_address_hash` without a `block_number` aren't extracted" do
assert Explorer.Indexer.AddressExtraction.extract_addresses(%{ assert Indexer.AddressExtraction.extract_addresses(%{
internal_transactions: [ internal_transactions: [
%{ %{
from_address_hash: "0x0000000000000000000000000000000000000001" from_address_hash: "0x0000000000000000000000000000000000000001"
@ -37,7 +37,7 @@ defmodule Explorer.Indexer.AddressExtractionTest do
end end
test "internal_transactions with a `to_address_hash` without a `block_number` aren't extracted" do test "internal_transactions with a `to_address_hash` without a `block_number` aren't extracted" do
assert Explorer.Indexer.AddressExtraction.extract_addresses(%{ assert Indexer.AddressExtraction.extract_addresses(%{
internal_transactions: [ internal_transactions: [
%{ %{
to_address_hash: "0x0000000000000000000000000000000000000002" to_address_hash: "0x0000000000000000000000000000000000000002"
@ -48,7 +48,7 @@ defmodule Explorer.Indexer.AddressExtractionTest do
test "internal_transactions with a `created_contract_address_hash` and `created_contract_code` " <> test "internal_transactions with a `created_contract_address_hash` and `created_contract_code` " <>
"without a `block_number` aren't extracted" do "without a `block_number` aren't extracted" do
assert Explorer.Indexer.AddressExtraction.extract_addresses(%{ assert Indexer.AddressExtraction.extract_addresses(%{
internal_transactions: [ internal_transactions: [
%{ %{
created_contract_address_hash: "0x0000000000000000000000000000000000000003", created_contract_address_hash: "0x0000000000000000000000000000000000000003",
@ -59,7 +59,7 @@ defmodule Explorer.Indexer.AddressExtractionTest do
end end
test "differing contract code is ignored" do test "differing contract code is ignored" do
assert Explorer.Indexer.AddressExtraction.extract_addresses(%{ assert Indexer.AddressExtraction.extract_addresses(%{
internal_transactions: [ internal_transactions: [
%{ %{
block_number: 1, block_number: 1,

@ -1,16 +1,16 @@
defmodule Explorer.Indexer.BlockFetcherTest do defmodule Indexer.BlockFetcherTest do
# `async: false` due to use of named GenServer # `async: false` due to use of named GenServer
use Explorer.DataCase, async: false use Explorer.DataCase, async: false
import ExUnit.CaptureLog import ExUnit.CaptureLog
alias Explorer.Chain.{Address, Block, Log, Transaction, Wei} alias Explorer.Chain.{Address, Block, Log, Transaction, Wei}
alias Explorer.Indexer
alias Explorer.Indexer.{ alias Indexer.{
AddressBalanceFetcher, AddressBalanceFetcher,
AddressBalanceFetcherCase, AddressBalanceFetcherCase,
BlockFetcher, BlockFetcher,
BufferedTask,
InternalTransactionFetcher, InternalTransactionFetcher,
InternalTransactionFetcherCase, InternalTransactionFetcherCase,
Sequence Sequence
@ -46,7 +46,7 @@ defmodule Explorer.Indexer.BlockFetcherTest do
assert Repo.aggregate(Block, :count, :hash) == 0 assert Repo.aggregate(Block, :count, :hash) == 0
start_supervised!({Task.Supervisor, name: Explorer.Indexer.TaskSupervisor}) start_supervised!({Task.Supervisor, name: Indexer.TaskSupervisor})
AddressBalanceFetcherCase.start_supervised!() AddressBalanceFetcherCase.start_supervised!()
InternalTransactionFetcherCase.start_supervised!() InternalTransactionFetcherCase.start_supervised!()
start_supervised!(BlockFetcher) start_supervised!(BlockFetcher)
@ -89,7 +89,7 @@ defmodule Explorer.Indexer.BlockFetcherTest do
@tag :capture_log @tag :capture_log
@heading "persisted counts" @heading "persisted counts"
test "without debug_logs", %{state: state} do test "without debug_logs", %{state: state} do
start_supervised!({Task.Supervisor, name: Explorer.Indexer.TaskSupervisor}) start_supervised!({Task.Supervisor, name: Indexer.TaskSupervisor})
AddressBalanceFetcherCase.start_supervised!() AddressBalanceFetcherCase.start_supervised!()
InternalTransactionFetcherCase.start_supervised!() InternalTransactionFetcherCase.start_supervised!()
@ -104,7 +104,7 @@ defmodule Explorer.Indexer.BlockFetcherTest do
@tag :capture_log @tag :capture_log
test "with debug_logs", %{state: state} do test "with debug_logs", %{state: state} do
start_supervised!({Task.Supervisor, name: Explorer.Indexer.TaskSupervisor}) start_supervised!({Task.Supervisor, name: Indexer.TaskSupervisor})
AddressBalanceFetcherCase.start_supervised!() AddressBalanceFetcherCase.start_supervised!()
InternalTransactionFetcherCase.start_supervised!() InternalTransactionFetcherCase.start_supervised!()
@ -129,7 +129,7 @@ defmodule Explorer.Indexer.BlockFetcherTest do
setup :state setup :state
setup do setup do
start_supervised!({Task.Supervisor, name: Explorer.Indexer.TaskSupervisor}) start_supervised!({Task.Supervisor, name: Indexer.TaskSupervisor})
AddressBalanceFetcherCase.start_supervised!() AddressBalanceFetcherCase.start_supervised!()
InternalTransactionFetcherCase.start_supervised!() InternalTransactionFetcherCase.start_supervised!()
{:ok, state} = BlockFetcher.init([]) {:ok, state} = BlockFetcher.init([])
@ -293,7 +293,7 @@ defmodule Explorer.Indexer.BlockFetcherTest do
defp wait_for_tasks(buffered_task) do defp wait_for_tasks(buffered_task) do
wait_until(5000, fn -> wait_until(5000, fn ->
counts = Explorer.BufferedTask.debug_count(buffered_task) counts = BufferedTask.debug_count(buffered_task)
counts.buffer == 0 and counts.tasks == 0 counts.buffer == 0 and counts.tasks == 0
end) end)
end end

@ -1,7 +1,7 @@
defmodule Explorer.BufferedTaskTest do defmodule Indexer.BufferedTaskTest do
use ExUnit.Case, async: true use ExUnit.Case, async: true
alias Explorer.BufferedTask alias Indexer.BufferedTask
@max_batch_size 2 @max_batch_size 2

@ -1,15 +1,15 @@
defmodule Explorer.Indexer.InternalTransactionFetcherTest do defmodule Indexer.InternalTransactionFetcherTest do
use Explorer.DataCase, async: false use Explorer.DataCase, async: false
import ExUnit.CaptureLog import ExUnit.CaptureLog
alias Explorer.Chain.Transaction alias Explorer.Chain.Transaction
alias Explorer.Indexer.{AddressBalanceFetcherCase, InternalTransactionFetcher, PendingTransactionFetcher} alias Indexer.{AddressBalanceFetcherCase, InternalTransactionFetcher, PendingTransactionFetcher}
@moduletag :capture_log @moduletag :capture_log
test "does not try to fetch pending transactions from Explorer.Indexer.PendingTransactionFetcher" do test "does not try to fetch pending transactions from Indexer.PendingTransactionFetcher" do
start_supervised!({Task.Supervisor, name: Explorer.Indexer.TaskSupervisor}) start_supervised!({Task.Supervisor, name: Indexer.TaskSupervisor})
AddressBalanceFetcherCase.start_supervised!() AddressBalanceFetcherCase.start_supervised!()
start_supervised!(PendingTransactionFetcher) start_supervised!(PendingTransactionFetcher)
@ -57,7 +57,7 @@ defmodule Explorer.Indexer.InternalTransactionFetcherTest do
describe "run/2" do describe "run/2" do
test "duplicate transaction hashes are logged" do test "duplicate transaction hashes are logged" do
start_supervised!({Task.Supervisor, name: Explorer.Indexer.TaskSupervisor}) start_supervised!({Task.Supervisor, name: Indexer.TaskSupervisor})
AddressBalanceFetcherCase.start_supervised!() AddressBalanceFetcherCase.start_supervised!()
insert(:transaction, hash: "0x03cd5899a63b6f6222afda8705d059fd5a7d126bcabe962fb654d9736e6bcafa") insert(:transaction, hash: "0x03cd5899a63b6f6222afda8705d059fd5a7d126bcabe962fb654d9736e6bcafa")
@ -82,7 +82,7 @@ defmodule Explorer.Indexer.InternalTransactionFetcherTest do
end end
test "duplicate transaction hashes only retry uniques" do test "duplicate transaction hashes only retry uniques" do
start_supervised!({Task.Supervisor, name: Explorer.Indexer.TaskSupervisor}) start_supervised!({Task.Supervisor, name: Indexer.TaskSupervisor})
AddressBalanceFetcherCase.start_supervised!() AddressBalanceFetcherCase.start_supervised!()
# not a real transaction hash, so that it fails # not a real transaction hash, so that it fails

@ -1,16 +1,16 @@
defmodule Explorer.Indexer.PendingTransactionFetcherTest do defmodule Indexer.PendingTransactionFetcherTest do
# `async: false` due to use of named GenServer # `async: false` due to use of named GenServer
use Explorer.DataCase, async: false use Explorer.DataCase, async: false
alias Explorer.Chain.Transaction alias Explorer.Chain.Transaction
alias Explorer.Indexer.PendingTransactionFetcher alias Indexer.PendingTransactionFetcher
describe "start_link/1" do describe "start_link/1" do
# this test may fail if Sokol so low volume that the pending transactions are empty for too long # this test may fail if Sokol so low volume that the pending transactions are empty for too long
test "starts fetching pending transactions" do test "starts fetching pending transactions" do
assert Repo.aggregate(Transaction, :count, :hash) == 0 assert Repo.aggregate(Transaction, :count, :hash) == 0
start_supervised!({Task.Supervisor, name: Explorer.Indexer.TaskSupervisor}) start_supervised!({Task.Supervisor, name: Indexer.TaskSupervisor})
start_supervised!(PendingTransactionFetcher) start_supervised!(PendingTransactionFetcher)
wait_for_results(fn -> wait_for_results(fn ->

@ -1,7 +1,7 @@
defmodule Explorer.Indexer.SequenceTest do defmodule Indexer.SequenceTest do
use ExUnit.Case use ExUnit.Case
alias Explorer.Indexer.Sequence alias Indexer.Sequence
test "start_link" do test "start_link" do
{:ok, pid} = Sequence.start_link([1..4], 5, 1) {:ok, pid} = Sequence.start_link([1..4], 5, 1)

@ -1,8 +1,6 @@
defmodule Explorer.IndexerTest do defmodule IndexerTest do
use Explorer.DataCase, async: true use Explorer.DataCase, async: true
alias Explorer.Indexer
import Explorer.Factory import Explorer.Factory
doctest Indexer doctest Indexer

@ -1,5 +1,5 @@
defmodule Explorer.Indexer.AddressBalanceFetcherCase do defmodule Indexer.AddressBalanceFetcherCase do
alias Explorer.Indexer.AddressBalanceFetcher alias Indexer.AddressBalanceFetcher
def start_supervised!(options \\ []) when is_list(options) do def start_supervised!(options \\ []) when is_list(options) do
options options

@ -1,5 +1,5 @@
defmodule Explorer.Indexer.InternalTransactionFetcherCase do defmodule Indexer.InternalTransactionFetcherCase do
alias Explorer.Indexer.InternalTransactionFetcher alias Indexer.InternalTransactionFetcher
def start_supervised!(options \\ []) when is_list(options) do def start_supervised!(options \\ []) when is_list(options) do
options options

@ -1 +1,20 @@
# https://github.com/CircleCI-Public/circleci-demo-elixir-phoenix/blob/a89de33a01df67b6773ac90adc74c34367a4a2d6/test/test_helper.exs#L1-L3
junit_folder = Mix.Project.build_path() <> "/junit/#{Mix.Project.config()[:app]}"
File.mkdir_p!(junit_folder)
:ok = Application.put_env(:junit_formatter, :report_dir, junit_folder)
# start all dependencies, but not Indexer itself as we need to unit test the supervision tree without and don't want the
# genesis task scanning in the background
Application.load(:indexer)
for application <- Application.spec(:indexer, :applications) do
Application.ensure_all_started(application)
end
# no declared in :applications since it is test-only
{:ok, _} = Application.ensure_all_started(:ex_machina)
ExUnit.configure(formatters: [JUnitFormatter, ExUnit.CLIFormatter])
ExUnit.start() ExUnit.start()
Ecto.Adapters.SQL.Sandbox.mode(Explorer.Repo, :manual)

@ -1,7 +1,7 @@
{ {
"coverage_options": { "coverage_options": {
"treat_no_relevant_lines_as_covered": true, "treat_no_relevant_lines_as_covered": true,
"minimum_coverage": 93.7 "minimum_coverage": 94.4
}, },
"terminal_options": { "terminal_options": {
"file_column_width": 120 "file_column_width": 120

@ -28,11 +28,19 @@ defmodule ExplorerUmbrella.Mixfile do
## Private Functions ## Private Functions
defp aliases(:dev) do defp aliases(env) do
[
# to match behavior of `mix test` in `apps/indexer`, which needs to not start applications for `indexer` to
# prevent its supervision tree from starting, which is undesirable in test
test: "test --no-start"
] ++ env_aliases(env)
end
defp env_aliases(:dev) do
[] []
end end
defp aliases(_env) do defp env_aliases(_env) do
[ [
compile: "compile --warnings-as-errors" compile: "compile --warnings-as-errors"
] ]

Loading…
Cancel
Save