Flatten indexer module hierarchy and supervisor tree (#1718)

* Add a defsupervisor macro with common supervisor functionality

This macro allows removing more than a dozen of copy-pasted `supervisor.ex`
modules.

The `Indexer.Fetcher` module will be expanded with even more common fetcher
functionality in upcoming PRs.

* Flatten indexer module hierarchy and remove generated supervisors

Modules are organized into 3 directories: `fetcher`, `transform` and
`temporary`. Supervisors are replaced with `use Indexer.Fetcher` clause.

`token_balances.ex` and block fetchers are left as-is for now, as they
require more invasive code refactoring, which will be done in upcoming PRs.

* Flatten supervisor tree

There is no need in separate `Block.Supervisor` hierarchy, so its children
are placed in the root supervisor tree.

The tree itself is moved to `Indexer.Supervisor` module to reduce amount
of directories.

* Add CHANGELOG.md entry
pull/1724/head
Paul Tsupikoff 6 years ago committed by GitHub
parent a432fb9503
commit c423a41e49
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 4
      CHANGELOG.md
  2. 4
      apps/block_scout_web/lib/block_scout_web/controllers/address_coin_balance_controller.ex
  3. 4
      apps/block_scout_web/lib/block_scout_web/controllers/address_contract_controller.ex
  4. 4
      apps/block_scout_web/lib/block_scout_web/controllers/address_decompiled_contract_controller.ex
  5. 4
      apps/block_scout_web/lib/block_scout_web/controllers/address_internal_transaction_controller.ex
  6. 4
      apps/block_scout_web/lib/block_scout_web/controllers/address_read_contract_controller.ex
  7. 4
      apps/block_scout_web/lib/block_scout_web/controllers/address_token_controller.ex
  8. 4
      apps/block_scout_web/lib/block_scout_web/controllers/address_token_transfer_controller.ex
  9. 4
      apps/block_scout_web/lib/block_scout_web/controllers/address_transaction_controller.ex
  10. 4
      apps/block_scout_web/lib/block_scout_web/controllers/address_validation_controller.ex
  11. 8
      apps/indexer/config/config.exs
  12. 9
      apps/indexer/lib/indexer/application.ex
  13. 12
      apps/indexer/lib/indexer/block/catchup/fetcher.ex
  14. 6
      apps/indexer/lib/indexer/block/catchup/sequence.ex
  15. 37
      apps/indexer/lib/indexer/block/fetcher.ex
  16. 11
      apps/indexer/lib/indexer/block/realtime/fetcher.ex
  17. 46
      apps/indexer/lib/indexer/block/reward/supervisor.ex
  18. 78
      apps/indexer/lib/indexer/block/supervisor.ex
  19. 21
      apps/indexer/lib/indexer/block/transform.ex
  20. 38
      apps/indexer/lib/indexer/block/uncle/supervisor.ex
  21. 39
      apps/indexer/lib/indexer/code/supervisor.ex
  22. 39
      apps/indexer/lib/indexer/coin_balance/supervisor.ex
  23. 77
      apps/indexer/lib/indexer/fetcher.ex
  24. 20
      apps/indexer/lib/indexer/fetcher/block_reward.ex
  25. 7
      apps/indexer/lib/indexer/fetcher/coin_balance.ex
  26. 9
      apps/indexer/lib/indexer/fetcher/coin_balance_on_demand.ex
  27. 14
      apps/indexer/lib/indexer/fetcher/contract_code.ex
  28. 12
      apps/indexer/lib/indexer/fetcher/internal_transaction.ex
  29. 24
      apps/indexer/lib/indexer/fetcher/pending_transaction.ex
  30. 9
      apps/indexer/lib/indexer/fetcher/replaced_transaction.ex
  31. 7
      apps/indexer/lib/indexer/fetcher/token.ex
  32. 7
      apps/indexer/lib/indexer/fetcher/token_balance.ex
  33. 7
      apps/indexer/lib/indexer/fetcher/token_updater.ex
  34. 23
      apps/indexer/lib/indexer/fetcher/uncle_block.ex
  35. 39
      apps/indexer/lib/indexer/internal_transaction/supervisor.ex
  36. 39
      apps/indexer/lib/indexer/pending_transaction/supervisor.ex
  37. 47
      apps/indexer/lib/indexer/replaced_transaction/supervisor.ex
  38. 97
      apps/indexer/lib/indexer/shrinkable/supervisor.ex
  39. 137
      apps/indexer/lib/indexer/supervisor.ex
  40. 1
      apps/indexer/lib/indexer/temporary/addresses_without_code.ex
  41. 38
      apps/indexer/lib/indexer/temporary/addresses_without_code/supervisor.ex
  42. 4
      apps/indexer/lib/indexer/temporary/failed_created_addresses.ex
  43. 38
      apps/indexer/lib/indexer/temporary/failed_created_addresses/supervisor.ex
  44. 7
      apps/indexer/lib/indexer/temporary/uncataloged_token_transfers.ex
  45. 41
      apps/indexer/lib/indexer/token/supervisor.ex
  46. 38
      apps/indexer/lib/indexer/token_balance/supervisor.ex
  47. 7
      apps/indexer/lib/indexer/token_balances.ex
  48. 44
      apps/indexer/lib/indexer/token_transfer/uncataloged/supervisor.ex
  49. 9
      apps/indexer/lib/indexer/token_transfers.ex
  50. 2
      apps/indexer/lib/indexer/transform/address_coin_balances.ex
  51. 2
      apps/indexer/lib/indexer/transform/address_token_balances.ex
  52. 14
      apps/indexer/lib/indexer/transform/addresses.ex
  53. 20
      apps/indexer/lib/indexer/transform/blocks.ex
  54. 8
      apps/indexer/lib/indexer/transform/blocks/base.ex
  55. 10
      apps/indexer/lib/indexer/transform/blocks/clique.ex
  56. 4
      apps/indexer/lib/indexer/transform/mint_transfers.ex
  57. 2
      apps/indexer/lib/indexer/transform/token_transfers.ex
  58. 24
      apps/indexer/test/indexer/block/catchup/bound_interval_supervisor_test.exs
  59. 9
      apps/indexer/test/indexer/block/catchup/fetcher_test.exs
  60. 4
      apps/indexer/test/indexer/block/catchup/sequence_test.exs
  61. 30
      apps/indexer/test/indexer/block/fetcher_test.exs
  62. 7
      apps/indexer/test/indexer/block/realtime/fetcher_test.exs
  63. 40
      apps/indexer/test/indexer/block/util_test.exs
  64. 48
      apps/indexer/test/indexer/fetcher/block_reward_test.exs
  65. 18
      apps/indexer/test/indexer/fetcher/coin_balance_on_demand_test.exs
  66. 14
      apps/indexer/test/indexer/fetcher/coin_balance_test.exs
  67. 8
      apps/indexer/test/indexer/fetcher/contract_code_test.exs
  68. 31
      apps/indexer/test/indexer/fetcher/internal_transaction_test.exs
  69. 4
      apps/indexer/test/indexer/fetcher/pending_transaction_test.exs
  70. 8
      apps/indexer/test/indexer/fetcher/replaced_transaction_test.exs
  71. 18
      apps/indexer/test/indexer/fetcher/token_balance_test.exs
  72. 8
      apps/indexer/test/indexer/fetcher/token_test.exs
  73. 8
      apps/indexer/test/indexer/fetcher/token_updater_test.exs
  74. 9
      apps/indexer/test/indexer/fetcher/uncle_block_test.exs
  75. 7
      apps/indexer/test/indexer/temporary/addresses_without_code_test.exs
  76. 2
      apps/indexer/test/indexer/temporary/failed_created_addresses_test.exs
  77. 33
      apps/indexer/test/indexer/temporary/uncataloged_token_transfers_test.exs
  78. 2
      apps/indexer/test/indexer/token_balances_test.exs
  79. 24
      apps/indexer/test/indexer/transform/address_coin_balances_test.exs
  80. 10
      apps/indexer/test/indexer/transform/address_token_balances_test.exs
  81. 40
      apps/indexer/test/indexer/transform/addresses.exs
  82. 4
      apps/indexer/test/indexer/transform/blocks/base_test.exs
  83. 4
      apps/indexer/test/indexer/transform/blocks/clique_test.exs
  84. 41
      apps/indexer/test/indexer/transform/blocks_test.exs
  85. 10
      apps/indexer/test/indexer/transform/mint_transfers_test.exs
  86. 10
      apps/indexer/test/indexer/transform/token_transfers_test.exs
  87. 0
      apps/indexer/test/support/indexer/block/catchup_supervisor_case.ex
  88. 6
      apps/indexer/test/support/indexer/fetcher/block_reward_supervisor_case.ex
  89. 4
      apps/indexer/test/support/indexer/fetcher/coin_balance_supervisor_case.ex
  90. 6
      apps/indexer/test/support/indexer/fetcher/contract_code_supervisor_case.ex
  91. 4
      apps/indexer/test/support/indexer/fetcher/internal_transaction_supervisor_case.ex
  92. 4
      apps/indexer/test/support/indexer/fetcher/pending_transaction_supervisor_case.ex
  93. 4
      apps/indexer/test/support/indexer/fetcher/replaced_transaction_supervisor_case.ex
  94. 4
      apps/indexer/test/support/indexer/fetcher/token_balance_supervisor_case.ex
  95. 4
      apps/indexer/test/support/indexer/fetcher/token_supervisor_case.ex
  96. 6
      apps/indexer/test/support/indexer/fetcher/uncle_block_supervisor_case.ex

@ -4,10 +4,12 @@
### Fixes ### Fixes
- [#1727](https://github.com/poanetwork/blockscout/pull/1727) - add logs pagination in rpc api - [#1727](https://github.com/poanetwork/blockscout/pull/1727) - add logs pagination in rpc api
### Chore ### Chore
- [#1718](https://github.com/poanetwork/blockscout/pull/1718) - Flatten indexer module hierarchy and supervisor tree
## 1.3.9-beta ## 1.3.9-beta

@ -11,7 +11,7 @@ defmodule BlockScoutWeb.AddressCoinBalanceController do
alias BlockScoutWeb.AddressCoinBalanceView alias BlockScoutWeb.AddressCoinBalanceView
alias Explorer.{Chain, Market} alias Explorer.{Chain, Market}
alias Explorer.ExchangeRates.Token alias Explorer.ExchangeRates.Token
alias Indexer.CoinBalance.OnDemandFetcher alias Indexer.Fetcher.CoinBalanceOnDemand
alias Phoenix.View alias Phoenix.View
def index(conn, %{"address_id" => address_hash_string, "type" => "JSON"} = params) do def index(conn, %{"address_id" => address_hash_string, "type" => "JSON"} = params) do
@ -62,7 +62,7 @@ defmodule BlockScoutWeb.AddressCoinBalanceController do
{:ok, address} <- Chain.hash_to_address(address_hash) do {:ok, address} <- Chain.hash_to_address(address_hash) do
render(conn, "index.html", render(conn, "index.html",
address: address, address: address,
coin_balance_status: OnDemandFetcher.trigger_fetch(address), coin_balance_status: CoinBalanceOnDemand.trigger_fetch(address),
exchange_rate: Market.get_exchange_rate(Explorer.coin()) || Token.null(), exchange_rate: Market.get_exchange_rate(Explorer.coin()) || Token.null(),
transaction_count: transaction_count(address), transaction_count: transaction_count(address),
validation_count: validation_count(address), validation_count: validation_count(address),

@ -5,7 +5,7 @@ defmodule BlockScoutWeb.AddressContractController do
alias Explorer.{Chain, Market} alias Explorer.{Chain, Market}
alias Explorer.ExchangeRates.Token alias Explorer.ExchangeRates.Token
alias Indexer.CoinBalance.OnDemandFetcher alias Indexer.Fetcher.CoinBalanceOnDemand
def index(conn, %{"address_id" => address_hash_string}) do def index(conn, %{"address_id" => address_hash_string}) do
with {:ok, address_hash} <- Chain.string_to_address_hash(address_hash_string), with {:ok, address_hash} <- Chain.string_to_address_hash(address_hash_string),
@ -14,7 +14,7 @@ defmodule BlockScoutWeb.AddressContractController do
conn, conn,
"index.html", "index.html",
address: address, address: address,
coin_balance_status: OnDemandFetcher.trigger_fetch(address), coin_balance_status: CoinBalanceOnDemand.trigger_fetch(address),
exchange_rate: Market.get_exchange_rate(Explorer.coin()) || Token.null(), exchange_rate: Market.get_exchange_rate(Explorer.coin()) || Token.null(),
transaction_count: transaction_count(address), transaction_count: transaction_count(address),
validation_count: validation_count(address) validation_count: validation_count(address)

@ -5,7 +5,7 @@ defmodule BlockScoutWeb.AddressDecompiledContractController do
alias Explorer.{Chain, Market} alias Explorer.{Chain, Market}
alias Explorer.ExchangeRates.Token alias Explorer.ExchangeRates.Token
alias Indexer.CoinBalance.OnDemandFetcher alias Indexer.Fetcher.CoinBalanceOnDemand
def index(conn, %{"address_id" => address_hash_string}) do def index(conn, %{"address_id" => address_hash_string}) do
with {:ok, address_hash} <- Chain.string_to_address_hash(address_hash_string), with {:ok, address_hash} <- Chain.string_to_address_hash(address_hash_string),
@ -14,7 +14,7 @@ defmodule BlockScoutWeb.AddressDecompiledContractController do
conn, conn,
"index.html", "index.html",
address: address, address: address,
coin_balance_status: OnDemandFetcher.trigger_fetch(address), coin_balance_status: CoinBalanceOnDemand.trigger_fetch(address),
exchange_rate: Market.get_exchange_rate(Explorer.coin()) || Token.null(), exchange_rate: Market.get_exchange_rate(Explorer.coin()) || Token.null(),
transaction_count: transaction_count(address), transaction_count: transaction_count(address),
validation_count: validation_count(address) validation_count: validation_count(address)

@ -11,7 +11,7 @@ defmodule BlockScoutWeb.AddressInternalTransactionController do
alias BlockScoutWeb.InternalTransactionView alias BlockScoutWeb.InternalTransactionView
alias Explorer.{Chain, Market} alias Explorer.{Chain, Market}
alias Explorer.ExchangeRates.Token alias Explorer.ExchangeRates.Token
alias Indexer.CoinBalance.OnDemandFetcher alias Indexer.Fetcher.CoinBalanceOnDemand
alias Phoenix.View alias Phoenix.View
def index(conn, %{"address_id" => address_hash_string, "type" => "JSON"} = params) do def index(conn, %{"address_id" => address_hash_string, "type" => "JSON"} = params) do
@ -67,7 +67,7 @@ defmodule BlockScoutWeb.AddressInternalTransactionController do
conn, conn,
"index.html", "index.html",
address: address, address: address,
coin_balance_status: OnDemandFetcher.trigger_fetch(address), coin_balance_status: CoinBalanceOnDemand.trigger_fetch(address),
current_path: current_path(conn), current_path: current_path(conn),
exchange_rate: Market.get_exchange_rate(Explorer.coin()) || Token.null(), exchange_rate: Market.get_exchange_rate(Explorer.coin()) || Token.null(),
filter: params["filter"], filter: params["filter"],

@ -10,7 +10,7 @@ defmodule BlockScoutWeb.AddressReadContractController do
alias Explorer.{Chain, Market} alias Explorer.{Chain, Market}
alias Explorer.ExchangeRates.Token alias Explorer.ExchangeRates.Token
alias Indexer.CoinBalance.OnDemandFetcher alias Indexer.Fetcher.CoinBalanceOnDemand
import BlockScoutWeb.AddressController, only: [transaction_count: 1, validation_count: 1] import BlockScoutWeb.AddressController, only: [transaction_count: 1, validation_count: 1]
@ -21,7 +21,7 @@ defmodule BlockScoutWeb.AddressReadContractController do
conn, conn,
"index.html", "index.html",
address: address, address: address,
coin_balance_status: OnDemandFetcher.trigger_fetch(address), coin_balance_status: CoinBalanceOnDemand.trigger_fetch(address),
exchange_rate: Market.get_exchange_rate(Explorer.coin()) || Token.null(), exchange_rate: Market.get_exchange_rate(Explorer.coin()) || Token.null(),
transaction_count: transaction_count(address), transaction_count: transaction_count(address),
validation_count: validation_count(address) validation_count: validation_count(address)

@ -3,7 +3,7 @@ defmodule BlockScoutWeb.AddressTokenController do
alias Explorer.{Chain, Market} alias Explorer.{Chain, Market}
alias Explorer.ExchangeRates.Token alias Explorer.ExchangeRates.Token
alias Indexer.CoinBalance.OnDemandFetcher alias Indexer.Fetcher.CoinBalanceOnDemand
import BlockScoutWeb.AddressController, only: [transaction_count: 1, validation_count: 1] import BlockScoutWeb.AddressController, only: [transaction_count: 1, validation_count: 1]
import BlockScoutWeb.Chain, only: [next_page_params: 3, paging_options: 1, split_list_by_page: 1] import BlockScoutWeb.Chain, only: [next_page_params: 3, paging_options: 1, split_list_by_page: 1]
@ -18,7 +18,7 @@ defmodule BlockScoutWeb.AddressTokenController do
conn, conn,
"index.html", "index.html",
address: address, address: address,
coin_balance_status: OnDemandFetcher.trigger_fetch(address), coin_balance_status: CoinBalanceOnDemand.trigger_fetch(address),
exchange_rate: Market.get_exchange_rate(Explorer.coin()) || Token.null(), exchange_rate: Market.get_exchange_rate(Explorer.coin()) || Token.null(),
transaction_count: transaction_count(address), transaction_count: transaction_count(address),
validation_count: validation_count(address), validation_count: validation_count(address),

@ -4,7 +4,7 @@ defmodule BlockScoutWeb.AddressTokenTransferController do
alias BlockScoutWeb.TransactionView alias BlockScoutWeb.TransactionView
alias Explorer.ExchangeRates.Token alias Explorer.ExchangeRates.Token
alias Explorer.{Chain, Market} alias Explorer.{Chain, Market}
alias Indexer.CoinBalance.OnDemandFetcher alias Indexer.Fetcher.CoinBalanceOnDemand
alias Phoenix.View alias Phoenix.View
import BlockScoutWeb.AddressController, only: [transaction_count: 1, validation_count: 1] import BlockScoutWeb.AddressController, only: [transaction_count: 1, validation_count: 1]
@ -81,7 +81,7 @@ defmodule BlockScoutWeb.AddressTokenTransferController do
conn, conn,
"index.html", "index.html",
address: address, address: address,
coin_balance_status: OnDemandFetcher.trigger_fetch(address), coin_balance_status: CoinBalanceOnDemand.trigger_fetch(address),
exchange_rate: Market.get_exchange_rate(Explorer.coin()) || Token.null(), exchange_rate: Market.get_exchange_rate(Explorer.coin()) || Token.null(),
current_path: current_path(conn), current_path: current_path(conn),
token: token, token: token,

@ -11,7 +11,7 @@ defmodule BlockScoutWeb.AddressTransactionController do
alias BlockScoutWeb.TransactionView alias BlockScoutWeb.TransactionView
alias Explorer.{Chain, Market} alias Explorer.{Chain, Market}
alias Explorer.ExchangeRates.Token alias Explorer.ExchangeRates.Token
alias Indexer.CoinBalance.OnDemandFetcher alias Indexer.Fetcher.CoinBalanceOnDemand
alias Phoenix.View alias Phoenix.View
@transaction_necessity_by_association [ @transaction_necessity_by_association [
@ -91,7 +91,7 @@ defmodule BlockScoutWeb.AddressTransactionController do
conn, conn,
"index.html", "index.html",
address: address, address: address,
coin_balance_status: OnDemandFetcher.trigger_fetch(address), coin_balance_status: CoinBalanceOnDemand.trigger_fetch(address),
exchange_rate: Market.get_exchange_rate(Explorer.coin()) || Token.null(), exchange_rate: Market.get_exchange_rate(Explorer.coin()) || Token.null(),
filter: params["filter"], filter: params["filter"],
transaction_count: transaction_count(address), transaction_count: transaction_count(address),

@ -12,7 +12,7 @@ defmodule BlockScoutWeb.AddressValidationController do
alias BlockScoutWeb.BlockView alias BlockScoutWeb.BlockView
alias Explorer.ExchangeRates.Token alias Explorer.ExchangeRates.Token
alias Explorer.{Chain, Market} alias Explorer.{Chain, Market}
alias Indexer.CoinBalance.OnDemandFetcher alias Indexer.Fetcher.CoinBalanceOnDemand
alias Phoenix.View alias Phoenix.View
def index(conn, %{"address_id" => address_hash_string, "type" => "JSON"} = params) do def index(conn, %{"address_id" => address_hash_string, "type" => "JSON"} = params) do
@ -76,7 +76,7 @@ defmodule BlockScoutWeb.AddressValidationController do
conn, conn,
"index.html", "index.html",
address: address, address: address,
coin_balance_status: OnDemandFetcher.trigger_fetch(address), coin_balance_status: CoinBalanceOnDemand.trigger_fetch(address),
current_path: current_path(conn), current_path: current_path(conn),
transaction_count: transaction_count(address), transaction_count: transaction_count(address),
validation_count: validation_count(address), validation_count: validation_count(address),

@ -5,8 +5,8 @@ use Mix.Config
import Bitwise import Bitwise
block_transformers = %{ block_transformers = %{
"clique" => Indexer.Block.Transform.Clique, "clique" => Indexer.Transform.Blocks.Clique,
"base" => Indexer.Block.Transform.Base "base" => Indexer.Transform.Blocks.Base
} }
# Compile time environment variable access requires recompilation. # Compile time environment variable access requires recompilation.
@ -36,8 +36,8 @@ config :indexer,
memory_limit: 1 <<< 30, memory_limit: 1 <<< 30,
first_block: System.get_env("FIRST_BLOCK") || 0 first_block: System.get_env("FIRST_BLOCK") || 0
# config :indexer, Indexer.ReplacedTransaction.Supervisor, disabled?: true # config :indexer, Indexer.Fetcher.ReplacedTransaction.Supervisor, disabled?: true
# config :indexer, Indexer.Block.Reward.Supervisor, disabled?: true # config :indexer, Indexer.Fetcher.BlockReward.Supervisor, disabled?: true
config :indexer, Indexer.Tracer, config :indexer, Indexer.Tracer,
service: :indexer, service: :indexer,

@ -5,10 +5,7 @@ defmodule Indexer.Application do
use Application use Application
alias Indexer.{ alias Indexer.Memory
Memory,
Shrinkable
}
@impl Application @impl Application
def start(_type, _args) do def start(_type, _args) do
@ -22,13 +19,13 @@ defmodule Indexer.Application do
children = [ children = [
{Memory.Monitor, [memory_monitor_options, [name: memory_monitor_name]]}, {Memory.Monitor, [memory_monitor_options, [name: memory_monitor_name]]},
{Shrinkable.Supervisor, [%{memory_monitor: memory_monitor_name}]} {Indexer.Supervisor, [%{memory_monitor: memory_monitor_name}]}
] ]
opts = [ opts = [
# If the `Memory.Monitor` dies, it needs all the `Shrinkable`s to re-register, so restart them. # If the `Memory.Monitor` dies, it needs all the `Shrinkable`s to re-register, so restart them.
strategy: :rest_for_one, strategy: :rest_for_one,
name: Indexer.Supervisor name: Indexer.Application
] ]
Supervisor.start_link(children, opts) Supervisor.start_link(children, opts)

@ -20,7 +20,9 @@ defmodule Indexer.Block.Catchup.Fetcher do
alias Ecto.Changeset alias Ecto.Changeset
alias Explorer.Chain alias Explorer.Chain
alias Explorer.Chain.{Hash, Transaction} alias Explorer.Chain.{Hash, Transaction}
alias Indexer.{Block, Code, InternalTransaction, Sequence, TokenBalance, Tracer} alias Indexer.{Block, Tracer}
alias Indexer.Block.Catchup.Sequence
alias Indexer.Fetcher.{ContractCode, InternalTransaction, TokenBalance}
alias Indexer.Memory.Shrinkable alias Indexer.Memory.Shrinkable
@behaviour Block.Fetcher @behaviour Block.Fetcher
@ -175,7 +177,7 @@ defmodule Indexer.Block.Catchup.Fetcher do
%Transaction{created_contract_address_hash: nil} -> %Transaction{created_contract_address_hash: nil} ->
[] []
end) end)
|> Code.Fetcher.async_fetch(10_000) |> ContractCode.async_fetch(10_000)
end end
defp async_import_created_contract_codes(_), do: :ok defp async_import_created_contract_codes(_), do: :ok
@ -183,7 +185,7 @@ defmodule Indexer.Block.Catchup.Fetcher do
defp async_import_internal_transactions(%{blocks: blocks}, EthereumJSONRPC.Parity) do defp async_import_internal_transactions(%{blocks: blocks}, EthereumJSONRPC.Parity) do
blocks blocks
|> Enum.map(fn %Chain.Block{number: block_number} -> %{number: block_number} end) |> Enum.map(fn %Chain.Block{number: block_number} -> %{number: block_number} end)
|> InternalTransaction.Fetcher.async_block_fetch(10_000) |> InternalTransaction.async_block_fetch(10_000)
end end
defp async_import_internal_transactions(%{transactions: transactions}, EthereumJSONRPC.Geth) do defp async_import_internal_transactions(%{transactions: transactions}, EthereumJSONRPC.Geth) do
@ -200,13 +202,13 @@ defmodule Indexer.Block.Catchup.Fetcher do
|> Enum.filter(fn %{block_number: block_number} -> |> Enum.filter(fn %{block_number: block_number} ->
max_block_number - block_number < @geth_block_limit max_block_number - block_number < @geth_block_limit
end) end)
|> InternalTransaction.Fetcher.async_fetch(10_000) |> InternalTransaction.async_fetch(10_000)
end end
defp async_import_internal_transactions(_, _), do: :ok defp async_import_internal_transactions(_, _), do: :ok
defp async_import_token_balances(%{address_token_balances: token_balances}) do defp async_import_token_balances(%{address_token_balances: token_balances}) do
TokenBalance.Fetcher.async_fetch(token_balances) TokenBalance.async_fetch(token_balances)
end end
defp async_import_token_balances(_), do: :ok defp async_import_token_balances(_), do: :ok

@ -1,4 +1,4 @@
defmodule Indexer.Sequence do defmodule Indexer.Block.Catchup.Sequence do
@moduledoc false @moduledoc false
use GenServer use GenServer
@ -71,11 +71,11 @@ defmodule Indexer.Sequence do
Infinite sequence Infinite sequence
Indexer.Sequence.start_link(first: 100, step: 10) Indexer.Block.Catchup.Sequence.start_link(first: 100, step: 10)
Finite sequence Finite sequence
Indexer.Sequence.start_link(ranges: [100..0]) Indexer.Block.Catchup.Sequence.start_link(ranges: [100..0])
""" """
@spec start_link(options(), Keyword.t()) :: GenServer.on_start() @spec start_link(options(), Keyword.t()) :: GenServer.on_start()

@ -11,10 +11,19 @@ defmodule Indexer.Block.Fetcher do
alias EthereumJSONRPC.{Blocks, FetchedBeneficiaries} alias EthereumJSONRPC.{Blocks, FetchedBeneficiaries}
alias Explorer.Chain.{Address, Block, Hash, Import, Transaction} alias Explorer.Chain.{Address, Block, Hash, Import, Transaction}
alias Indexer.{AddressExtraction, CoinBalance, MintTransfer, ReplacedTransaction, Token, TokenTransfers, Tracer}
alias Indexer.Address.{CoinBalances, TokenBalances}
alias Indexer.Block.Fetcher.Receipts alias Indexer.Block.Fetcher.Receipts
alias Indexer.Block.{Reward, Transform} alias Indexer.Fetcher.{BlockReward, CoinBalance, ReplacedTransaction, Token, UncleBlock}
alias Indexer.Tracer
alias Indexer.Transform.{
AddressCoinBalances,
Addresses,
AddressTokenBalances,
MintTransfers,
TokenTransfers
}
alias Indexer.Transform.Blocks, as: TransformBlocks
@type address_hash_to_fetched_balance_block_number :: %{String.t() => Block.block_number()} @type address_hash_to_fetched_balance_block_number :: %{String.t() => Block.block_number()}
@ -102,16 +111,16 @@ defmodule Indexer.Block.Fetcher do
block_second_degree_relations_params: block_second_degree_relations_params, block_second_degree_relations_params: block_second_degree_relations_params,
errors: blocks_errors errors: blocks_errors
}}} <- {:blocks, EthereumJSONRPC.fetch_blocks_by_range(range, json_rpc_named_arguments)}, }}} <- {:blocks, EthereumJSONRPC.fetch_blocks_by_range(range, json_rpc_named_arguments)},
blocks = Transform.transform_blocks(blocks_params), blocks = TransformBlocks.transform_blocks(blocks_params),
{:receipts, {:ok, receipt_params}} <- {:receipts, Receipts.fetch(state, transactions_params_without_receipts)}, {:receipts, {:ok, receipt_params}} <- {:receipts, Receipts.fetch(state, transactions_params_without_receipts)},
%{logs: logs, receipts: receipts} = receipt_params, %{logs: logs, receipts: receipts} = receipt_params,
transactions_with_receipts = Receipts.put(transactions_params_without_receipts, receipts), transactions_with_receipts = Receipts.put(transactions_params_without_receipts, receipts),
%{token_transfers: token_transfers, tokens: tokens} = TokenTransfers.parse(logs), %{token_transfers: token_transfers, tokens: tokens} = TokenTransfers.parse(logs),
%{mint_transfers: mint_transfers} = MintTransfer.parse(logs), %{mint_transfers: mint_transfers} = MintTransfers.parse(logs),
%FetchedBeneficiaries{params_set: beneficiary_params_set, errors: beneficiaries_errors} = %FetchedBeneficiaries{params_set: beneficiary_params_set, errors: beneficiaries_errors} =
fetch_beneficiaries(blocks, json_rpc_named_arguments), fetch_beneficiaries(blocks, json_rpc_named_arguments),
addresses = addresses =
AddressExtraction.extract_addresses(%{ Addresses.extract_addresses(%{
block_reward_contract_beneficiaries: MapSet.to_list(beneficiary_params_set), block_reward_contract_beneficiaries: MapSet.to_list(beneficiary_params_set),
blocks: blocks, blocks: blocks,
logs: logs, logs: logs,
@ -126,12 +135,12 @@ defmodule Indexer.Block.Fetcher do
logs_params: logs, logs_params: logs,
transactions_params: transactions_with_receipts transactions_params: transactions_with_receipts
} }
|> CoinBalances.params_set(), |> AddressCoinBalances.params_set(),
beneficiaries_with_gas_payment <- beneficiaries_with_gas_payment <-
beneficiary_params_set beneficiary_params_set
|> add_gas_payments(transactions_with_receipts) |> add_gas_payments(transactions_with_receipts)
|> Reward.Fetcher.reduce_uncle_rewards(), |> BlockReward.reduce_uncle_rewards(),
address_token_balances = TokenBalances.params_set(%{token_transfers_params: token_transfers}), address_token_balances = AddressTokenBalances.params_set(%{token_transfers_params: token_transfers}),
{:ok, inserted} <- {:ok, inserted} <-
__MODULE__.import( __MODULE__.import(
state, state,
@ -180,7 +189,7 @@ defmodule Indexer.Block.Fetcher do
def async_import_block_rewards(errors) when is_list(errors) do def async_import_block_rewards(errors) when is_list(errors) do
errors errors
|> block_reward_errors_to_block_numbers() |> block_reward_errors_to_block_numbers()
|> Indexer.Block.Reward.Fetcher.async_fetch() |> BlockReward.async_fetch()
end end
def async_import_coin_balances(%{addresses: addresses}, %{ def async_import_coin_balances(%{addresses: addresses}, %{
@ -191,7 +200,7 @@ defmodule Indexer.Block.Fetcher do
block_number = Map.fetch!(address_hash_to_block_number, to_string(address_hash)) block_number = Map.fetch!(address_hash_to_block_number, to_string(address_hash))
%{address_hash: address_hash, block_number: block_number} %{address_hash: address_hash, block_number: block_number}
end) end)
|> CoinBalance.Fetcher.async_fetch_balances() |> CoinBalance.async_fetch_balances()
end end
def async_import_coin_balances(_, _), do: :ok def async_import_coin_balances(_, _), do: :ok
@ -199,7 +208,7 @@ defmodule Indexer.Block.Fetcher do
def async_import_tokens(%{tokens: tokens}) do def async_import_tokens(%{tokens: tokens}) do
tokens tokens
|> Enum.map(& &1.contract_address_hash) |> Enum.map(& &1.contract_address_hash)
|> Token.Fetcher.async_fetch() |> Token.async_fetch()
end end
def async_import_tokens(_), do: :ok def async_import_tokens(_), do: :ok
@ -207,7 +216,7 @@ defmodule Indexer.Block.Fetcher do
def async_import_uncles(%{block_second_degree_relations: block_second_degree_relations}) do def async_import_uncles(%{block_second_degree_relations: block_second_degree_relations}) do
block_second_degree_relations block_second_degree_relations
|> Enum.map(& &1.uncle_hash) |> Enum.map(& &1.uncle_hash)
|> Indexer.Block.Uncle.Fetcher.async_fetch_blocks() |> UncleBlock.async_fetch_blocks()
end end
def async_import_uncles(_), do: :ok def async_import_uncles(_), do: :ok
@ -221,7 +230,7 @@ defmodule Indexer.Block.Fetcher do
%Transaction{block_hash: nil} -> %Transaction{block_hash: nil} ->
[] []
end) end)
|> ReplacedTransaction.Fetcher.async_fetch(10_000) |> ReplacedTransaction.async_fetch(10_000)
end end
def async_import_replaced_transactions(_), do: :ok def async_import_replaced_transactions(_), do: :ok

@ -26,8 +26,9 @@ defmodule Indexer.Block.Realtime.Fetcher do
alias Explorer.Chain alias Explorer.Chain
alias Explorer.Chain.TokenTransfer alias Explorer.Chain.TokenTransfer
alias Explorer.Counters.AverageBlockTime alias Explorer.Counters.AverageBlockTime
alias Indexer.{AddressExtraction, Block, TokenBalances, Tracer} alias Indexer.{Block, TokenBalances, Tracer}
alias Indexer.Block.Realtime.TaskSupervisor alias Indexer.Block.Realtime.TaskSupervisor
alias Indexer.Transform.Addresses
alias Timex.Duration alias Timex.Duration
@behaviour Block.Fetcher @behaviour Block.Fetcher
@ -394,9 +395,9 @@ defmodule Indexer.Block.Realtime.Fetcher do
{:ok, internal_transactions_params} -> {:ok, internal_transactions_params} ->
merged_addresses_params = merged_addresses_params =
%{internal_transactions: internal_transactions_params} %{internal_transactions: internal_transactions_params}
|> AddressExtraction.extract_addresses() |> Addresses.extract_addresses()
|> Kernel.++(addresses_params) |> Kernel.++(addresses_params)
|> AddressExtraction.merge_addresses() |> Addresses.merge_addresses()
{:ok, {:ok,
%{ %{
@ -505,9 +506,9 @@ defmodule Indexer.Block.Realtime.Fetcher do
{:ok, %FetchedBalances{params_list: params_list, errors: []}} -> {:ok, %FetchedBalances{params_list: params_list, errors: []}} ->
merged_addresses_params = merged_addresses_params =
%{address_coin_balances: params_list} %{address_coin_balances: params_list}
|> AddressExtraction.extract_addresses() |> Addresses.extract_addresses()
|> Kernel.++(addresses_params) |> Kernel.++(addresses_params)
|> AddressExtraction.merge_addresses() |> Addresses.merge_addresses()
value_fetched_at = DateTime.utc_now() value_fetched_at = DateTime.utc_now()

@ -1,46 +0,0 @@
defmodule Indexer.Block.Reward.Supervisor do
@moduledoc """
Supervises `Indexer.Block.Reward.Fetcher` and its batch tasks through `Indexer.Block.Reward.TaskSupervisor`
"""
use Supervisor
alias Indexer.Block.Reward.Fetcher
def child_spec([init_arguments]) do
child_spec([init_arguments, []])
end
def child_spec([_init_arguments, _gen_server_options] = start_link_arguments) do
default = %{
id: __MODULE__,
start: {__MODULE__, :start_link, start_link_arguments},
type: :supervisor
}
Supervisor.child_spec(default, [])
end
def start_link(arguments, gen_server_options \\ []) do
if disabled?() do
:ignore
else
Supervisor.start_link(__MODULE__, arguments, Keyword.put_new(gen_server_options, :name, __MODULE__))
end
end
def disabled?() do
Application.get_env(:indexer, __MODULE__, [])[:disabled?] == true
end
@impl Supervisor
def init(fetcher_arguments) do
Supervisor.init(
[
{Task.Supervisor, name: Indexer.Block.Reward.TaskSupervisor},
{Fetcher, [fetcher_arguments, [name: Fetcher]]}
],
strategy: :one_for_one
)
end
end

@ -1,78 +0,0 @@
defmodule Indexer.Block.Supervisor do
@moduledoc """
Supervises `Indexer.Block.Catchup.Supervisor` and `Indexer.Block.Realtime.Supervisor`.
"""
alias Indexer.Block
alias Indexer.Block.{Catchup, Realtime, Reward, Uncle}
alias Indexer.Temporary.{AddressesWithoutCode, FailedCreatedAddresses}
use Supervisor
def start_link([arguments, gen_server_options]) do
Supervisor.start_link(__MODULE__, arguments, gen_server_options)
end
@impl Supervisor
def init(
%{
block_interval: block_interval,
json_rpc_named_arguments: json_rpc_named_arguments,
subscribe_named_arguments: subscribe_named_arguments,
realtime_overrides: realtime_overrides
} = named_arguments
) do
block_fetcher =
named_arguments
|> Map.drop(~w(block_interval memory_monitor subscribe_named_arguments realtime_overrides)a)
|> Block.Fetcher.new()
fixing_realtime_fetcher = %Block.Fetcher{
broadcast: false,
callback_module: Realtime.Fetcher,
json_rpc_named_arguments: json_rpc_named_arguments
}
realtime_block_fetcher =
named_arguments
|> Map.drop(~w(block_interval memory_monitor subscribe_named_arguments realtime_overrides)a)
|> Map.merge(Enum.into(realtime_overrides, %{}))
|> Block.Fetcher.new()
realtime_subscribe_named_arguments = realtime_overrides[:subscribe_named_arguments] || subscribe_named_arguments
memory_monitor = Map.get(named_arguments, :memory_monitor)
Supervisor.init(
[
{Catchup.Supervisor,
[
%{block_fetcher: block_fetcher, block_interval: block_interval, memory_monitor: memory_monitor},
[name: Catchup.Supervisor]
]},
{Realtime.Supervisor,
[
%{block_fetcher: realtime_block_fetcher, subscribe_named_arguments: realtime_subscribe_named_arguments},
[name: Realtime.Supervisor]
]},
{Uncle.Supervisor, [[block_fetcher: block_fetcher, memory_monitor: memory_monitor], [name: Uncle.Supervisor]]},
{Reward.Supervisor,
[
[json_rpc_named_arguments: json_rpc_named_arguments, memory_monitor: memory_monitor],
[name: Reward.Supervisor]
]},
{FailedCreatedAddresses.Supervisor,
[
json_rpc_named_arguments,
[name: FailedCreatedAddresses.Supervisor]
]},
{AddressesWithoutCode.Supervisor,
[
fixing_realtime_fetcher,
[name: AddressesWithoutCode.Supervisor]
]}
],
strategy: :one_for_one
)
end
end

@ -1,21 +0,0 @@
defmodule Indexer.Block.Transform do
@moduledoc """
Protocol for transforming blocks.
"""
@type block :: map()
@doc """
Transforms a block.
"""
@callback transform(block :: block()) :: block()
@doc """
Runs a list of blocks through the configured block transformer.
"""
def transform_blocks(blocks) when is_list(blocks) do
transformer = Application.get_env(:indexer, :block_transformer)
Enum.map(blocks, &transformer.transform/1)
end
end

@ -1,38 +0,0 @@
defmodule Indexer.Block.Uncle.Supervisor do
@moduledoc """
Supervises `Indexer.Block.Uncle.Fetcher`.
"""
use Supervisor
alias Indexer.Block.Uncle.Fetcher
def child_spec([init_arguments]) do
child_spec([init_arguments, []])
end
def child_spec([_init_arguments, _gen_server_options] = start_link_arguments) do
default = %{
id: __MODULE__,
start: {__MODULE__, :start_link, start_link_arguments},
type: :supervisor
}
Supervisor.child_spec(default, [])
end
def start_link(arguments, gen_server_options \\ []) do
Supervisor.start_link(__MODULE__, arguments, gen_server_options)
end
@impl Supervisor
def init(fetcher_arguments) do
Supervisor.init(
[
{Task.Supervisor, name: Indexer.Block.Uncle.TaskSupervisor},
{Fetcher, [fetcher_arguments, [name: Fetcher]]}
],
strategy: :rest_for_one
)
end
end

@ -1,39 +0,0 @@
defmodule Indexer.Code.Supervisor do
@moduledoc """
Supervises `Indexer.Code.Fetcher` and its batch tasks through
`Indexer.Code.TaskSupervisor`.
"""
use Supervisor
alias Indexer.Code.Fetcher
def child_spec([init_arguments]) do
child_spec([init_arguments, []])
end
def child_spec([_init_arguments, _gen_server_options] = start_link_arguments) do
default = %{
id: __MODULE__,
start: {__MODULE__, :start_link, start_link_arguments},
type: :supervisor
}
Supervisor.child_spec(default, [])
end
def start_link(arguments, gen_server_options \\ []) do
Supervisor.start_link(__MODULE__, arguments, Keyword.put_new(gen_server_options, :name, __MODULE__))
end
@impl Supervisor
def init(fetcher_arguments) do
Supervisor.init(
[
{Task.Supervisor, name: Indexer.Code.TaskSupervisor},
{Fetcher, [fetcher_arguments, [name: Fetcher]]}
],
strategy: :one_for_one
)
end
end

@ -1,39 +0,0 @@
defmodule Indexer.CoinBalance.Supervisor do
@moduledoc """
Supervises `Indexer.CoinBalance.Fetcher` and its batch tasks through `Indexer.CoinBalance.TaskSupervisor`
"""
use Supervisor
alias Indexer.CoinBalance.{Fetcher, OnDemandFetcher}
def child_spec([init_arguments]) do
child_spec([init_arguments, []])
end
def child_spec([_init_arguments, _gen_server_options] = start_link_arguments) do
default = %{
id: __MODULE__,
start: {__MODULE__, :start_link, start_link_arguments},
type: :supervisor
}
Supervisor.child_spec(default, [])
end
def start_link(arguments, gen_server_options \\ []) do
Supervisor.start_link(__MODULE__, arguments, Keyword.put_new(gen_server_options, :name, __MODULE__))
end
@impl Supervisor
def init(fetcher_arguments) do
Supervisor.init(
[
{Task.Supervisor, name: Indexer.CoinBalance.TaskSupervisor},
{Fetcher, [fetcher_arguments, [name: Fetcher]]},
{OnDemandFetcher, [fetcher_arguments[:json_rpc_named_arguments], [name: OnDemandFetcher]]}
],
strategy: :one_for_one
)
end
end

@ -0,0 +1,77 @@
defmodule Indexer.Fetcher do
@moduledoc """
General fetcher infrastructure.
"""
alias Macro.Env
defmacro __using__(opts \\ []) do
quote do
require Indexer.Fetcher
Indexer.Fetcher.defsupervisor(unquote(opts))
end
end
# credo:disable-for-next-line Credo.Check.Refactor.CyclomaticComplexity
defmacro defsupervisor(opts \\ []) do
quote location: :keep do
opts = unquote(opts)
strategy = Keyword.get(opts, :strategy, :one_for_one)
fetcher = __MODULE__
supervisor = Keyword.get(opts, :supervisor, Module.concat(fetcher, Supervisor))
task_supervisor = Keyword.get(opts, :task_supervisor, Module.concat(fetcher, TaskSupervisor))
Module.create(
supervisor,
quote bind_quoted: [strategy: strategy, fetcher: fetcher, task_supervisor: task_supervisor] do
use Supervisor
def child_spec([]), do: child_spec([[], []])
def child_spec([init_arguments]), do: child_spec([init_arguments, []])
def child_spec([_init_arguments, _gen_server_options] = start_link_arguments) do
default = %{
id: __MODULE__,
start: {__MODULE__, :start_link, start_link_arguments},
restart: :transient,
type: :supervisor
}
Supervisor.child_spec(default, [])
end
def start_link(arguments, gen_server_options \\ []) do
if disabled?() do
:ignore
else
Supervisor.start_link(__MODULE__, arguments, Keyword.put_new(gen_server_options, :name, __MODULE__))
end
end
def disabled?() do
Application.get_env(:indexer, __MODULE__, [])[:disabled?] == true
end
@impl Supervisor
def init(fetcher_arguments) do
children = [
{Task.Supervisor, name: unquote(task_supervisor)},
{unquote(fetcher), [put_supervisor_when_is_list(fetcher_arguments), [name: unquote(fetcher)]]}
]
Supervisor.init(children, strategy: unquote(strategy))
end
defp put_supervisor_when_is_list(arguments) when is_list(arguments) do
Keyword.put(arguments, :supervisor, self())
end
defp put_supervisor_when_is_list(arguments), do: arguments
end,
Env.location(__ENV__)
)
end
end
end

@ -1,4 +1,4 @@
defmodule Indexer.Block.Reward.Fetcher do defmodule Indexer.Fetcher.BlockReward do
@moduledoc """ @moduledoc """
Fetches `t:Explorer.Chain.Block.Reward.t/0` for a given `t:Explorer.Chain.Block.block_number/0`. Fetches `t:Explorer.Chain.Block.Reward.t/0` for a given `t:Explorer.Chain.Block.block_number/0`.
@ -6,6 +6,7 @@ defmodule Indexer.Block.Reward.Fetcher do
retrieved from the database and compared against that returned from `EthereumJSONRPC.` retrieved from the database and compared against that returned from `EthereumJSONRPC.`
""" """
use Indexer.Fetcher
use Spandex.Decorators use Spandex.Decorators
require Logger require Logger
@ -16,9 +17,10 @@ defmodule Indexer.Block.Reward.Fetcher do
alias EthereumJSONRPC.FetchedBeneficiaries alias EthereumJSONRPC.FetchedBeneficiaries
alias Explorer.Chain alias Explorer.Chain
alias Explorer.Chain.{Block, Wei} alias Explorer.Chain.{Block, Wei}
alias Indexer.Address.CoinBalances alias Indexer.{BufferedTask, Tracer}
alias Indexer.{AddressExtraction, BufferedTask, CoinBalance, Tracer} alias Indexer.Fetcher.BlockReward.Supervisor, as: BlockRewardSupervisor
alias Indexer.Block.Reward.Supervisor, as: BlockRewardSupervisor alias Indexer.Fetcher.CoinBalance
alias Indexer.Transform.{AddressCoinBalances, Addresses}
@behaviour BufferedTask @behaviour BufferedTask
@ -26,7 +28,7 @@ defmodule Indexer.Block.Reward.Fetcher do
flush_interval: :timer.seconds(3), flush_interval: :timer.seconds(3),
max_batch_size: 10, max_batch_size: 10,
max_concurrency: 4, max_concurrency: 4,
task_supervisor: Indexer.Block.Reward.TaskSupervisor, task_supervisor: Indexer.Fetcher.BlockReward.TaskSupervisor,
metadata: [fetcher: :block_reward] metadata: [fetcher: :block_reward]
] ]
@ -72,7 +74,7 @@ defmodule Indexer.Block.Reward.Fetcher do
end end
@impl BufferedTask @impl BufferedTask
@decorate trace(name: "fetch", resource: "Indexer.Block.Reward.Fetcher.run/2", service: :indexer, tracer: Tracer) @decorate trace(name: "fetch", resource: "Indexer.Fetcher.BlockReward.run/2", service: :indexer, tracer: Tracer)
def run(entries, json_rpc_named_arguments) do def run(entries, json_rpc_named_arguments) do
hash_string_by_number = hash_string_by_number =
entries entries
@ -129,7 +131,7 @@ defmodule Indexer.Block.Reward.Fetcher do
|> import_block_reward_params() |> import_block_reward_params()
|> case do |> case do
{:ok, %{address_coin_balances: address_coin_balances}} -> {:ok, %{address_coin_balances: address_coin_balances}} ->
CoinBalance.Fetcher.async_fetch_balances(address_coin_balances) CoinBalance.async_fetch_balances(address_coin_balances)
retry_errors(errors) retry_errors(errors)
@ -237,8 +239,8 @@ defmodule Indexer.Block.Reward.Fetcher do
end end
defp import_block_reward_params(block_rewards_params) when is_list(block_rewards_params) do defp import_block_reward_params(block_rewards_params) when is_list(block_rewards_params) do
addresses_params = AddressExtraction.extract_addresses(%{block_reward_contract_beneficiaries: block_rewards_params}) addresses_params = Addresses.extract_addresses(%{block_reward_contract_beneficiaries: block_rewards_params})
address_coin_balances_params_set = CoinBalances.params_set(%{beneficiary_params: block_rewards_params}) address_coin_balances_params_set = AddressCoinBalances.params_set(%{beneficiary_params: block_rewards_params})
Chain.import(%{ Chain.import(%{
addresses: %{params: addresses_params}, addresses: %{params: addresses_params},

@ -1,9 +1,10 @@
defmodule Indexer.CoinBalance.Fetcher do defmodule Indexer.Fetcher.CoinBalance do
@moduledoc """ @moduledoc """
Fetches `t:Explorer.Chain.Address.CoinBalance.t/0` and updates `t:Explorer.Chain.Address.t/0` `fetched_coin_balance` and Fetches `t:Explorer.Chain.Address.CoinBalance.t/0` and updates `t:Explorer.Chain.Address.t/0` `fetched_coin_balance` and
`fetched_coin_balance_block_number` to value at max `t:Explorer.Chain.Address.CoinBalance.t/0` `block_number` for the given `t:Explorer.Chain.Address.t/` `hash`. `fetched_coin_balance_block_number` to value at max `t:Explorer.Chain.Address.CoinBalance.t/0` `block_number` for the given `t:Explorer.Chain.Address.t/` `hash`.
""" """
use Indexer.Fetcher
use Spandex.Decorators use Spandex.Decorators
require Logger require Logger
@ -21,7 +22,7 @@ defmodule Indexer.CoinBalance.Fetcher do
flush_interval: :timer.seconds(3), flush_interval: :timer.seconds(3),
max_batch_size: 500, max_batch_size: 500,
max_concurrency: 4, max_concurrency: 4,
task_supervisor: Indexer.CoinBalance.TaskSupervisor, task_supervisor: Indexer.Fetcher.CoinBalance.TaskSupervisor,
metadata: [fetcher: :coin_balance] metadata: [fetcher: :coin_balance]
] ]
@ -69,7 +70,7 @@ defmodule Indexer.CoinBalance.Fetcher do
end end
@impl BufferedTask @impl BufferedTask
@decorate trace(name: "fetch", resource: "Indexer.CoinBalance.Fetcher.run/2", service: :indexer, tracer: Tracer) @decorate trace(name: "fetch", resource: "Indexer.Fetcher.CoinBalance.run/2", service: :indexer, tracer: Tracer)
def run(entries, json_rpc_named_arguments) do def run(entries, json_rpc_named_arguments) do
# the same address may be used more than once in the same block, but we only want one `Balance` for a given # the same address may be used more than once in the same block, but we only want one `Balance` for a given
# `{address, block}`, so take unique params only # `{address, block}`, so take unique params only

@ -1,4 +1,4 @@
defmodule Indexer.CoinBalance.OnDemandFetcher do defmodule Indexer.Fetcher.CoinBalanceOnDemand do
@moduledoc """ @moduledoc """
Ensures that we have a reasonably up to date coin balance for a given address. Ensures that we have a reasonably up to date coin balance for a given address.
@ -10,6 +10,7 @@ defmodule Indexer.CoinBalance.OnDemandFetcher do
@latest_balance_stale_threshold :timer.hours(24) @latest_balance_stale_threshold :timer.hours(24)
use GenServer use GenServer
use Indexer.Fetcher
import Ecto.Query, only: [from: 2] import Ecto.Query, only: [from: 2]
import EthereumJSONRPC, only: [integer_to_quantity: 1] import EthereumJSONRPC, only: [integer_to_quantity: 1]
@ -19,7 +20,7 @@ defmodule Indexer.CoinBalance.OnDemandFetcher do
alias Explorer.Chain.{Address, BlockNumberCache} alias Explorer.Chain.{Address, BlockNumberCache}
alias Explorer.Chain.Address.CoinBalance alias Explorer.Chain.Address.CoinBalance
alias Explorer.Counters.AverageBlockTime alias Explorer.Counters.AverageBlockTime
alias Indexer.CoinBalance.Fetcher alias Indexer.Fetcher.CoinBalance, as: CoinBalanceFetcher
alias Timex.Duration alias Timex.Duration
@type block_number :: integer @type block_number :: integer
@ -134,7 +135,7 @@ defmodule Indexer.CoinBalance.OnDemandFetcher do
:ok :ok
{:ok, %{params_list: params_list}} -> {:ok, %{params_list: params_list}} ->
address_params = Fetcher.balances_params_to_address_params(params_list) address_params = CoinBalanceFetcher.balances_params_to_address_params(params_list)
Chain.import(%{ Chain.import(%{
addresses: %{params: address_params, with: :balance_changeset}, addresses: %{params: address_params, with: :balance_changeset},
@ -153,7 +154,7 @@ defmodule Indexer.CoinBalance.OnDemandFetcher do
end end
defp do_import(%FetchedBalances{} = fetched_balances) do defp do_import(%FetchedBalances{} = fetched_balances) do
case Fetcher.import_fetched_balances(fetched_balances, :on_demand) do case CoinBalanceFetcher.import_fetched_balances(fetched_balances, :on_demand) do
{:ok, %{addresses: [address]}} -> {:ok, address} {:ok, %{addresses: [address]}} -> {:ok, address}
_ -> :error _ -> :error
end end

@ -1,8 +1,9 @@
defmodule Indexer.Code.Fetcher do defmodule Indexer.Fetcher.ContractCode do
@moduledoc """ @moduledoc """
Fetches `contract_code` `t:Explorer.Chain.Address.t/0`. Fetches `contract_code` `t:Explorer.Chain.Address.t/0`.
""" """
use Indexer.Fetcher
use Spandex.Decorators use Spandex.Decorators
require Logger require Logger
@ -11,7 +12,8 @@ defmodule Indexer.Code.Fetcher do
alias Explorer.Chain alias Explorer.Chain
alias Explorer.Chain.{Block, Hash} alias Explorer.Chain.{Block, Hash}
alias Indexer.{AddressExtraction, BufferedTask, Tracer} alias Indexer.{BufferedTask, Tracer}
alias Indexer.Transform.Addresses
@behaviour BufferedTask @behaviour BufferedTask
@ -21,7 +23,7 @@ defmodule Indexer.Code.Fetcher do
flush_interval: :timer.seconds(3), flush_interval: :timer.seconds(3),
max_concurrency: @max_concurrency, max_concurrency: @max_concurrency,
max_batch_size: @max_batch_size, max_batch_size: @max_batch_size,
task_supervisor: Indexer.Code.TaskSupervisor, task_supervisor: Indexer.Fetcher.ContractCode.TaskSupervisor,
metadata: [fetcher: :code] metadata: [fetcher: :code]
] ]
@ -85,7 +87,7 @@ defmodule Indexer.Code.Fetcher do
@impl BufferedTask @impl BufferedTask
@decorate trace( @decorate trace(
name: "fetch", name: "fetch",
resource: "Indexer.Code.Fetcher.run/2", resource: "Indexer.Fetcher.ContractCode.run/2",
service: :indexer, service: :indexer,
tracer: Tracer tracer: Tracer
) )
@ -97,7 +99,7 @@ defmodule Indexer.Code.Fetcher do
|> EthereumJSONRPC.fetch_codes(json_rpc_named_arguments) |> EthereumJSONRPC.fetch_codes(json_rpc_named_arguments)
|> case do |> case do
{:ok, create_address_codes} -> {:ok, create_address_codes} ->
addresses_params = AddressExtraction.extract_addresses(%{codes: create_address_codes.params_list}) addresses_params = Addresses.extract_addresses(%{codes: create_address_codes.params_list})
import_with_balances(addresses_params, entries, json_rpc_named_arguments) import_with_balances(addresses_params, entries, json_rpc_named_arguments)
@ -118,7 +120,7 @@ defmodule Indexer.Code.Fetcher do
{:ok, fetched_balances} -> {:ok, fetched_balances} ->
balance_addresses_params = balances_params_to_address_params(fetched_balances.params_list) balance_addresses_params = balances_params_to_address_params(fetched_balances.params_list)
merged_addresses_params = AddressExtraction.merge_addresses(addresses_params ++ balance_addresses_params) merged_addresses_params = Addresses.merge_addresses(addresses_params ++ balance_addresses_params)
case Chain.import(%{ case Chain.import(%{
addresses: %{params: merged_addresses_params}, addresses: %{params: merged_addresses_params},

@ -1,10 +1,11 @@
defmodule Indexer.InternalTransaction.Fetcher do defmodule Indexer.Fetcher.InternalTransaction do
@moduledoc """ @moduledoc """
Fetches and indexes `t:Explorer.Chain.InternalTransaction.t/0`. Fetches and indexes `t:Explorer.Chain.InternalTransaction.t/0`.
See `async_fetch/1` for details on configuring limits. See `async_fetch/1` for details on configuring limits.
""" """
use Indexer.Fetcher
use Spandex.Decorators use Spandex.Decorators
require Logger require Logger
@ -13,7 +14,8 @@ defmodule Indexer.InternalTransaction.Fetcher do
alias Explorer.Chain alias Explorer.Chain
alias Explorer.Chain.{Block, Hash} alias Explorer.Chain.{Block, Hash}
alias Indexer.{AddressExtraction, BufferedTask, Tracer} alias Indexer.{BufferedTask, Tracer}
alias Indexer.Transform.Addresses
@behaviour BufferedTask @behaviour BufferedTask
@ -23,7 +25,7 @@ defmodule Indexer.InternalTransaction.Fetcher do
flush_interval: :timer.seconds(3), flush_interval: :timer.seconds(3),
max_concurrency: @max_concurrency, max_concurrency: @max_concurrency,
max_batch_size: @max_batch_size, max_batch_size: @max_batch_size,
task_supervisor: Indexer.InternalTransaction.TaskSupervisor, task_supervisor: Indexer.Fetcher.InternalTransaction.TaskSupervisor,
metadata: [fetcher: :internal_transaction] metadata: [fetcher: :internal_transaction]
] ]
@ -141,7 +143,7 @@ defmodule Indexer.InternalTransaction.Fetcher do
@impl BufferedTask @impl BufferedTask
@decorate trace( @decorate trace(
name: "fetch", name: "fetch",
resource: "Indexer.InternalTransaction.Fetcher.run/2", resource: "Indexer.Fetcher.InternalTransaction.run/2",
service: :indexer, service: :indexer,
tracer: Tracer tracer: Tracer
) )
@ -177,7 +179,7 @@ defmodule Indexer.InternalTransaction.Fetcher do
internal_transactions_params_without_failed_creations = remove_failed_creations(internal_transactions_params) internal_transactions_params_without_failed_creations = remove_failed_creations(internal_transactions_params)
addresses_params = addresses_params =
AddressExtraction.extract_addresses(%{ Addresses.extract_addresses(%{
internal_transactions: internal_transactions_params_without_failed_creations internal_transactions: internal_transactions_params_without_failed_creations
}) })

@ -1,4 +1,4 @@
defmodule Indexer.PendingTransaction.Fetcher do defmodule Indexer.Fetcher.PendingTransaction do
@moduledoc """ @moduledoc """
Fetches pending transactions and imports them. Fetches pending transactions and imports them.
@ -6,6 +6,7 @@ defmodule Indexer.PendingTransaction.Fetcher do
validated version that may make it to the database first. validated version that may make it to the database first.
""" """
use GenServer use GenServer
use Indexer.Fetcher
require Logger require Logger
@ -13,7 +14,8 @@ defmodule Indexer.PendingTransaction.Fetcher do
alias Ecto.Changeset alias Ecto.Changeset
alias Explorer.Chain alias Explorer.Chain
alias Indexer.{AddressExtraction, PendingTransaction} alias Indexer.Fetcher.PendingTransaction
alias Indexer.Transform.Addresses
@chunk_size 250 @chunk_size 250
@ -70,7 +72,7 @@ defmodule Indexer.PendingTransaction.Fetcher do
|> Keyword.merge(opts) |> Keyword.merge(opts)
state = state =
%PendingTransaction.Fetcher{ %__MODULE__{
json_rpc_named_arguments: Keyword.fetch!(opts, :json_rpc_named_arguments), json_rpc_named_arguments: Keyword.fetch!(opts, :json_rpc_named_arguments),
interval: opts[:pending_transaction_interval] || @default_interval interval: opts[:pending_transaction_interval] || @default_interval
} }
@ -80,12 +82,12 @@ defmodule Indexer.PendingTransaction.Fetcher do
end end
@impl GenServer @impl GenServer
def handle_info(:fetch, %PendingTransaction.Fetcher{} = state) do def handle_info(:fetch, %__MODULE__{} = state) do
task = Task.Supervisor.async_nolink(PendingTransaction.TaskSupervisor, fn -> task(state) end) task = Task.Supervisor.async_nolink(PendingTransaction.TaskSupervisor, fn -> task(state) end)
{:noreply, %PendingTransaction.Fetcher{state | task: task}} {:noreply, %__MODULE__{state | task: task}}
end end
def handle_info({ref, result}, %PendingTransaction.Fetcher{task: %Task{ref: ref}} = state) do def handle_info({ref, result}, %__MODULE__{task: %Task{ref: ref}} = state) do
Process.demonitor(ref, [:flush]) Process.demonitor(ref, [:flush])
case result do case result do
@ -99,19 +101,19 @@ defmodule Indexer.PendingTransaction.Fetcher do
def handle_info( def handle_info(
{:DOWN, ref, :process, pid, reason}, {:DOWN, ref, :process, pid, reason},
%PendingTransaction.Fetcher{task: %Task{pid: pid, ref: ref}} = state %__MODULE__{task: %Task{pid: pid, ref: ref}} = state
) do ) do
Logger.error(fn -> "pending transaction fetcher task exited due to #{inspect(reason)}. Rescheduling." end) Logger.error(fn -> "pending transaction fetcher task exited due to #{inspect(reason)}. Rescheduling." end)
{:noreply, schedule_fetch(state)} {:noreply, schedule_fetch(state)}
end end
defp schedule_fetch(%PendingTransaction.Fetcher{interval: interval} = state) do defp schedule_fetch(%__MODULE__{interval: interval} = state) do
Process.send_after(self(), :fetch, interval) Process.send_after(self(), :fetch, interval)
%PendingTransaction.Fetcher{state | task: nil} %__MODULE__{state | task: nil}
end end
defp task(%PendingTransaction.Fetcher{json_rpc_named_arguments: json_rpc_named_arguments} = _state) do defp task(%__MODULE__{json_rpc_named_arguments: json_rpc_named_arguments} = _state) do
Logger.metadata(fetcher: :pending_transaction) Logger.metadata(fetcher: :pending_transaction)
case fetch_pending_transactions(json_rpc_named_arguments) do case fetch_pending_transactions(json_rpc_named_arguments) do
@ -136,7 +138,7 @@ defmodule Indexer.PendingTransaction.Fetcher do
end end
defp import_chunk(transactions_params) do defp import_chunk(transactions_params) do
addresses_params = AddressExtraction.extract_addresses(%{transactions: transactions_params}, pending: true) addresses_params = Addresses.extract_addresses(%{transactions: transactions_params}, pending: true)
# There's no need to queue up fetching the address balance since theses are pending transactions and cannot have # There's no need to queue up fetching the address balance since theses are pending transactions and cannot have
# affected the address balance yet since address balance is a balance at a given block and these transactions are # affected the address balance yet since address balance is a balance at a given block and these transactions are

@ -1,8 +1,9 @@
defmodule Indexer.ReplacedTransaction.Fetcher do defmodule Indexer.Fetcher.ReplacedTransaction do
@moduledoc """ @moduledoc """
Finds and updates replaced transactions. Finds and updates replaced transactions.
""" """
use Indexer.Fetcher
use Spandex.Decorators use Spandex.Decorators
require Logger require Logger
@ -10,7 +11,7 @@ defmodule Indexer.ReplacedTransaction.Fetcher do
alias Explorer.Chain alias Explorer.Chain
alias Explorer.Chain.Hash alias Explorer.Chain.Hash
alias Indexer.{BufferedTask, Tracer} alias Indexer.{BufferedTask, Tracer}
alias Indexer.ReplacedTransaction.Supervisor, as: ReplacedTransactionSupervisor alias Indexer.Fetcher.ReplacedTransaction.Supervisor, as: ReplacedTransactionSupervisor
@behaviour BufferedTask @behaviour BufferedTask
@ -20,7 +21,7 @@ defmodule Indexer.ReplacedTransaction.Fetcher do
flush_interval: :timer.seconds(3), flush_interval: :timer.seconds(3),
max_concurrency: @max_concurrency, max_concurrency: @max_concurrency,
max_batch_size: @max_batch_size, max_batch_size: @max_batch_size,
task_supervisor: Indexer.ReplacedTransaction.TaskSupervisor, task_supervisor: Indexer.Fetcher.ReplacedTransaction.TaskSupervisor,
metadata: [fetcher: :replaced_transaction] metadata: [fetcher: :replaced_transaction]
] ]
@ -94,7 +95,7 @@ defmodule Indexer.ReplacedTransaction.Fetcher do
@impl BufferedTask @impl BufferedTask
@decorate trace( @decorate trace(
name: "fetch", name: "fetch",
resource: "Indexer.ReplacedTransaction.Fetcher.run/2", resource: "Indexer.Fetcher.ReplacedTransaction.run/2",
service: :indexer, service: :indexer,
tracer: Tracer tracer: Tracer
) )

@ -1,8 +1,9 @@
defmodule Indexer.Token.Fetcher do defmodule Indexer.Fetcher.Token do
@moduledoc """ @moduledoc """
Fetches information about a token. Fetches information about a token.
""" """
use Indexer.Fetcher
use Spandex.Decorators use Spandex.Decorators
alias Explorer.Chain alias Explorer.Chain
@ -17,7 +18,7 @@ defmodule Indexer.Token.Fetcher do
flush_interval: 300, flush_interval: 300,
max_batch_size: 1, max_batch_size: 1,
max_concurrency: 10, max_concurrency: 10,
task_supervisor: Indexer.Token.TaskSupervisor task_supervisor: Indexer.Fetcher.Token.TaskSupervisor
] ]
@doc false @doc false
@ -49,7 +50,7 @@ defmodule Indexer.Token.Fetcher do
end end
@impl BufferedTask @impl BufferedTask
@decorate trace(name: "fetch", resource: "Indexer.Token.Fetcher.run/2", service: :indexer, tracer: Tracer) @decorate trace(name: "fetch", resource: "Indexer.Fetcher.Token.run/2", service: :indexer, tracer: Tracer)
def run([token_contract_address], _json_rpc_named_arguments) do def run([token_contract_address], _json_rpc_named_arguments) do
case Chain.token_from_address_hash(token_contract_address) do case Chain.token_from_address_hash(token_contract_address) do
{:ok, %Token{} = token} -> {:ok, %Token{} = token} ->

@ -1,4 +1,4 @@
defmodule Indexer.TokenBalance.Fetcher do defmodule Indexer.Fetcher.TokenBalance do
@moduledoc """ @moduledoc """
Fetches token balances and send the ones that were fetched to be imported in `Address.CurrentTokenBalance` and Fetches token balances and send the ones that were fetched to be imported in `Address.CurrentTokenBalance` and
`Address.TokenBalance`. `Address.TokenBalance`.
@ -13,6 +13,7 @@ defmodule Indexer.TokenBalance.Fetcher do
that always raise errors interacting with the Smart Contract. that always raise errors interacting with the Smart Contract.
""" """
use Indexer.Fetcher
use Spandex.Decorators use Spandex.Decorators
require Logger require Logger
@ -27,7 +28,7 @@ defmodule Indexer.TokenBalance.Fetcher do
flush_interval: 300, flush_interval: 300,
max_batch_size: 100, max_batch_size: 100,
max_concurrency: 10, max_concurrency: 10,
task_supervisor: Indexer.TokenBalance.TaskSupervisor task_supervisor: Indexer.Fetcher.TokenBalance.TaskSupervisor
] ]
@max_retries 3 @max_retries 3
@ -75,7 +76,7 @@ defmodule Indexer.TokenBalance.Fetcher do
when reading their balance in the Smart Contract. when reading their balance in the Smart Contract.
""" """
@impl BufferedTask @impl BufferedTask
@decorate trace(name: "fetch", resource: "Indexer.TokenBalance.Fetcher.run/2", tracer: Tracer, service: :indexer) @decorate trace(name: "fetch", resource: "Indexer.Fetcher.TokenBalance.run/2", tracer: Tracer, service: :indexer)
def run(entries, _json_rpc_named_arguments) do def run(entries, _json_rpc_named_arguments) do
result = result =
entries entries

@ -1,16 +1,17 @@
defmodule Indexer.Token.MetadataUpdater do defmodule Indexer.Fetcher.TokenUpdater do
@moduledoc """ @moduledoc """
Updates metadata for cataloged tokens Updates metadata for cataloged tokens
""" """
use GenServer use GenServer
use Indexer.Fetcher
alias Explorer.Chain alias Explorer.Chain
alias Explorer.Chain.Token alias Explorer.Chain.Token
alias Explorer.Token.MetadataRetriever alias Explorer.Token.MetadataRetriever
def start_link(initial_state) do def start_link([initial_state, gen_server_options]) do
GenServer.start_link(__MODULE__, initial_state, name: __MODULE__) GenServer.start_link(__MODULE__, initial_state, gen_server_options)
end end
@impl true @impl true

@ -1,9 +1,10 @@
defmodule Indexer.Block.Uncle.Fetcher do defmodule Indexer.Fetcher.UncleBlock do
@moduledoc """ @moduledoc """
Fetches `t:Explorer.Chain.Block.t/0` by `hash` and updates `t:Explorer.Chain.Block.SecondDegreeRelation.t/0` Fetches `t:Explorer.Chain.Block.t/0` by `hash` and updates `t:Explorer.Chain.Block.SecondDegreeRelation.t/0`
`uncle_fetched_at` where the `uncle_hash` matches `hash`. `uncle_fetched_at` where the `uncle_hash` matches `hash`.
""" """
use Indexer.Fetcher
use Spandex.Decorators use Spandex.Decorators
require Logger require Logger
@ -12,7 +13,9 @@ defmodule Indexer.Block.Uncle.Fetcher do
alias EthereumJSONRPC.Blocks alias EthereumJSONRPC.Blocks
alias Explorer.Chain alias Explorer.Chain
alias Explorer.Chain.Hash alias Explorer.Chain.Hash
alias Indexer.{AddressExtraction, Block, BufferedTask, Tracer} alias Indexer.{Block, BufferedTask, Tracer}
alias Indexer.Fetcher.UncleBlock
alias Indexer.Transform.Addresses
@behaviour Block.Fetcher @behaviour Block.Fetcher
@behaviour BufferedTask @behaviour BufferedTask
@ -21,7 +24,7 @@ defmodule Indexer.Block.Uncle.Fetcher do
flush_interval: :timer.seconds(3), flush_interval: :timer.seconds(3),
max_batch_size: 10, max_batch_size: 10,
max_concurrency: 10, max_concurrency: 10,
task_supervisor: Indexer.Block.Uncle.TaskSupervisor, task_supervisor: Indexer.Fetcher.UncleBlock.TaskSupervisor,
metadata: [fetcher: :block_uncle] metadata: [fetcher: :block_uncle]
] ]
@ -70,7 +73,7 @@ defmodule Indexer.Block.Uncle.Fetcher do
end end
@impl BufferedTask @impl BufferedTask
@decorate trace(name: "fetch", resource: "Indexer.Block.Uncle.Fetcher.run/2", service: :indexer, tracer: Tracer) @decorate trace(name: "fetch", resource: "Indexer.Fetcher.UncleBlock.run/2", service: :indexer, tracer: Tracer)
def run(hashes, %Block.Fetcher{json_rpc_named_arguments: json_rpc_named_arguments} = block_fetcher) do def run(hashes, %Block.Fetcher{json_rpc_named_arguments: json_rpc_named_arguments} = block_fetcher) do
# the same block could be included as an uncle on multiple blocks, but we only want to fetch it once # the same block could be included as an uncle on multiple blocks, but we only want to fetch it once
unique_hashes = Enum.uniq(hashes) unique_hashes = Enum.uniq(hashes)
@ -108,7 +111,7 @@ defmodule Indexer.Block.Uncle.Fetcher do
block_fetcher, block_fetcher,
original_entries original_entries
) do ) do
addresses_params = AddressExtraction.extract_addresses(%{blocks: blocks_params, transactions: transactions_params}) addresses_params = Addresses.extract_addresses(%{blocks: blocks_params, transactions: transactions_params})
case Block.Fetcher.import(block_fetcher, %{ case Block.Fetcher.import(block_fetcher, %{
addresses: %{params: addresses_params}, addresses: %{params: addresses_params},
@ -149,15 +152,15 @@ defmodule Indexer.Block.Uncle.Fetcher do
|> uncle_blocks() |> uncle_blocks()
|> fork_transactions() |> fork_transactions()
|> Chain.import() do |> Chain.import() do
# * CoinBalance.Fetcher.async_fetch_balances is not called because uncles don't affect balances # * CoinBalance.async_fetch_balances is not called because uncles don't affect balances
# * InternalTransaction.Fetcher.async_fetch is not called because internal transactions are based on transaction # * InternalTransaction.async_fetch is not called because internal transactions are based on transaction
# hash, which is shared with transaction on consensus blocks. # hash, which is shared with transaction on consensus blocks.
# * Token.Fetcher.async_fetch is not called because the tokens only matter on consensus blocks # * Token.async_fetch is not called because the tokens only matter on consensus blocks
# * TokenBalance.Fetcher.async_fetch is not called because it uses block numbers from consensus, not uncles # * TokenBalance.async_fetch is not called because it uses block numbers from consensus, not uncles
block_second_degree_relations block_second_degree_relations
|> Enum.map(& &1.uncle_hash) |> Enum.map(& &1.uncle_hash)
|> Block.Uncle.Fetcher.async_fetch_blocks() |> UncleBlock.async_fetch_blocks()
ok ok
end end

@ -1,39 +0,0 @@
defmodule Indexer.InternalTransaction.Supervisor do
@moduledoc """
Supervises `Indexer.InternalTransaction.Fetcher` and its batch tasks through
`Indexer.InternalTransaction.TaskSupervisor`.
"""
use Supervisor
alias Indexer.InternalTransaction.Fetcher
def child_spec([init_arguments]) do
child_spec([init_arguments, []])
end
def child_spec([_init_arguments, _gen_server_options] = start_link_arguments) do
default = %{
id: __MODULE__,
start: {__MODULE__, :start_link, start_link_arguments},
type: :supervisor
}
Supervisor.child_spec(default, [])
end
def start_link(arguments, gen_server_options \\ []) do
Supervisor.start_link(__MODULE__, arguments, Keyword.put_new(gen_server_options, :name, __MODULE__))
end
@impl Supervisor
def init(fetcher_arguments) do
Supervisor.init(
[
{Task.Supervisor, name: Indexer.InternalTransaction.TaskSupervisor},
{Fetcher, [fetcher_arguments, [name: Fetcher]]}
],
strategy: :one_for_one
)
end
end

@ -1,39 +0,0 @@
defmodule Indexer.PendingTransaction.Supervisor do
@moduledoc """
Supervises `Indexer.PendingTransaction.Fetcher` and its batch tasks through
`Indexer.PendingTransaction.TaskSupervisor`.
"""
use Supervisor
alias Indexer.PendingTransaction.Fetcher
def child_spec([init_arguments]) do
child_spec([init_arguments, []])
end
def child_spec([_init_arguments, _gen_server_options] = start_link_arguments) do
default = %{
id: __MODULE__,
start: {__MODULE__, :start_link, start_link_arguments},
type: :supervisor
}
Supervisor.child_spec(default, [])
end
def start_link(arguments, gen_server_options \\ []) do
Supervisor.start_link(__MODULE__, arguments, Keyword.put_new(gen_server_options, :name, __MODULE__))
end
@impl Supervisor
def init(fetcher_arguments) do
Supervisor.init(
[
{Task.Supervisor, name: Indexer.PendingTransaction.TaskSupervisor},
{Fetcher, [fetcher_arguments, [name: Fetcher]]}
],
strategy: :one_for_one
)
end
end

@ -1,47 +0,0 @@
defmodule Indexer.ReplacedTransaction.Supervisor do
@moduledoc """
Supervises `Indexer.ReplacedTransaction.Fetcher` and its batch tasks through
`Indexer.ReplacedTransaction.TaskSupervisor`.
"""
use Supervisor
alias Indexer.ReplacedTransaction.Fetcher
def child_spec([init_arguments]) do
child_spec([init_arguments, []])
end
def child_spec([_init_arguments, _gen_server_options] = start_link_arguments) do
default = %{
id: __MODULE__,
start: {__MODULE__, :start_link, start_link_arguments},
type: :supervisor
}
Supervisor.child_spec(default, [])
end
def start_link(arguments, gen_server_options \\ []) do
if disabled?() do
:ignore
else
Supervisor.start_link(__MODULE__, arguments, Keyword.put_new(gen_server_options, :name, __MODULE__))
end
end
def disabled?() do
Application.get_env(:indexer, __MODULE__, [])[:disabled?] == true
end
@impl Supervisor
def init(fetcher_arguments) do
Supervisor.init(
[
{Task.Supervisor, name: Indexer.ReplacedTransaction.TaskSupervisor},
{Fetcher, [fetcher_arguments, [name: Fetcher]]}
],
strategy: :one_for_one
)
end
end

@ -1,97 +0,0 @@
defmodule Indexer.Shrinkable.Supervisor do
@moduledoc """
Supervisor of all supervision trees that depend on `Indexer.Alarm.Supervisor`.
"""
use Supervisor
alias Indexer.{
Block,
Code,
CoinBalance,
InternalTransaction,
PendingTransaction,
ReplacedTransaction,
Token,
TokenBalance,
TokenTransfer
}
def child_spec([]) do
child_spec([[]])
end
def child_spec([init_arguments]) do
child_spec([init_arguments, []])
end
def child_spec([_init_arguments, _gen_server_options] = start_link_arguments) do
default = %{
id: __MODULE__,
start: {__MODULE__, :start_link, start_link_arguments},
type: :supervisor
}
Supervisor.child_spec(default, [])
end
def start_link(arguments, gen_server_options \\ []) do
Supervisor.start_link(__MODULE__, arguments, Keyword.put_new(gen_server_options, :name, __MODULE__))
end
@impl Supervisor
def init(%{memory_monitor: memory_monitor}) do
json_rpc_named_arguments = Application.fetch_env!(:indexer, :json_rpc_named_arguments)
block_fetcher_supervisor_named_arguments =
:indexer
|> Application.get_all_env()
|> Keyword.take(
~w(blocks_batch_size blocks_concurrency block_interval json_rpc_named_arguments receipts_batch_size
receipts_concurrency subscribe_named_arguments realtime_overrides)a
)
|> Enum.into(%{})
|> Map.put(:memory_monitor, memory_monitor)
|> Map.put_new(:realtime_overrides, %{})
Supervisor.init(
[
{CoinBalance.Supervisor,
[
[json_rpc_named_arguments: json_rpc_named_arguments, memory_monitor: memory_monitor],
[name: CoinBalance.Supervisor]
]},
{PendingTransaction.Supervisor,
[[json_rpc_named_arguments: json_rpc_named_arguments], [name: PendingTransactionFetcher]]},
{Code.Supervisor,
[
[json_rpc_named_arguments: json_rpc_named_arguments, memory_monitor: memory_monitor],
[name: Code.Supervisor]
]},
{ReplacedTransaction.Supervisor,
[
[memory_monitor: memory_monitor],
[name: ReplacedTransaction.Supervisor]
]},
{InternalTransaction.Supervisor,
[
[json_rpc_named_arguments: json_rpc_named_arguments, memory_monitor: memory_monitor],
[name: InternalTransaction.Supervisor]
]},
{Token.Supervisor,
[
[json_rpc_named_arguments: json_rpc_named_arguments, memory_monitor: memory_monitor],
[name: Token.Supervisor]
]},
{TokenBalance.Supervisor,
[
[json_rpc_named_arguments: json_rpc_named_arguments, memory_monitor: memory_monitor],
[name: TokenBalance.Supervisor]
]},
{Block.Supervisor, [block_fetcher_supervisor_named_arguments, [name: Block.Supervisor]]},
{TokenTransfer.Uncataloged.Supervisor, [[], [name: TokenTransfer.Uncataloged.Supervisor]]}
],
strategy: :one_for_one
)
end
end

@ -0,0 +1,137 @@
defmodule Indexer.Supervisor do
@moduledoc """
Supervisor of all indexer worker supervision trees
"""
use Supervisor
alias Indexer.Block
alias Indexer.Block.{Catchup, Realtime}
alias Indexer.Fetcher.{
BlockReward,
CoinBalance,
CoinBalanceOnDemand,
ContractCode,
InternalTransaction,
PendingTransaction,
ReplacedTransaction,
Token,
TokenBalance,
TokenUpdater,
UncleBlock
}
alias Indexer.Temporary.{
AddressesWithoutCode,
FailedCreatedAddresses,
UncatalogedTokenTransfers
}
def child_spec([]) do
child_spec([[]])
end
def child_spec([init_arguments]) do
child_spec([init_arguments, []])
end
def child_spec([_init_arguments, _gen_server_options] = start_link_arguments) do
default = %{
id: __MODULE__,
start: {__MODULE__, :start_link, start_link_arguments},
type: :supervisor
}
Supervisor.child_spec(default, [])
end
def start_link(arguments, gen_server_options \\ []) do
Supervisor.start_link(__MODULE__, arguments, Keyword.put_new(gen_server_options, :name, __MODULE__))
end
@impl Supervisor
def init(%{memory_monitor: memory_monitor}) do
json_rpc_named_arguments = Application.fetch_env!(:indexer, :json_rpc_named_arguments)
named_arguments =
:indexer
|> Application.get_all_env()
|> Keyword.take(
~w(blocks_batch_size blocks_concurrency block_interval json_rpc_named_arguments receipts_batch_size
receipts_concurrency subscribe_named_arguments realtime_overrides)a
)
|> Enum.into(%{})
|> Map.put(:memory_monitor, memory_monitor)
|> Map.put_new(:realtime_overrides, %{})
%{
block_interval: block_interval,
realtime_overrides: realtime_overrides,
subscribe_named_arguments: subscribe_named_arguments
} = named_arguments
metadata_updater_inverval = Application.get_env(:indexer, :metadata_updater_days_interval)
block_fetcher =
named_arguments
|> Map.drop(~w(block_interval memory_monitor subscribe_named_arguments realtime_overrides)a)
|> Block.Fetcher.new()
fixing_realtime_fetcher = %Block.Fetcher{
broadcast: false,
callback_module: Realtime.Fetcher,
json_rpc_named_arguments: json_rpc_named_arguments
}
realtime_block_fetcher =
named_arguments
|> Map.drop(~w(block_interval memory_monitor subscribe_named_arguments realtime_overrides)a)
|> Map.merge(Enum.into(realtime_overrides, %{}))
|> Block.Fetcher.new()
realtime_subscribe_named_arguments = realtime_overrides[:subscribe_named_arguments] || subscribe_named_arguments
Supervisor.init(
[
# Root fetchers
{PendingTransaction.Supervisor, [[json_rpc_named_arguments: json_rpc_named_arguments]]},
{Realtime.Supervisor,
[
%{block_fetcher: realtime_block_fetcher, subscribe_named_arguments: realtime_subscribe_named_arguments},
[name: Realtime.Supervisor]
]},
{Catchup.Supervisor,
[
%{block_fetcher: block_fetcher, block_interval: block_interval, memory_monitor: memory_monitor},
[name: Catchup.Supervisor]
]},
# Async catchup fetchers
{UncleBlock.Supervisor, [[block_fetcher: block_fetcher, memory_monitor: memory_monitor]]},
{BlockReward.Supervisor,
[[json_rpc_named_arguments: json_rpc_named_arguments, memory_monitor: memory_monitor]]},
{InternalTransaction.Supervisor,
[[json_rpc_named_arguments: json_rpc_named_arguments, memory_monitor: memory_monitor]]},
{CoinBalance.Supervisor,
[[json_rpc_named_arguments: json_rpc_named_arguments, memory_monitor: memory_monitor]]},
{Token.Supervisor, [[json_rpc_named_arguments: json_rpc_named_arguments, memory_monitor: memory_monitor]]},
{ContractCode.Supervisor,
[[json_rpc_named_arguments: json_rpc_named_arguments, memory_monitor: memory_monitor]]},
{TokenBalance.Supervisor,
[[json_rpc_named_arguments: json_rpc_named_arguments, memory_monitor: memory_monitor]]},
{ReplacedTransaction.Supervisor, [[memory_monitor: memory_monitor]]},
# Out-of-band fetchers
{CoinBalanceOnDemand.Supervisor, [json_rpc_named_arguments]},
{TokenUpdater.Supervisor, [%{update_interval: metadata_updater_inverval}]},
# Temporary workers
{AddressesWithoutCode.Supervisor, [fixing_realtime_fetcher]},
{FailedCreatedAddresses.Supervisor, [json_rpc_named_arguments]},
{UncatalogedTokenTransfers.Supervisor, [[]]}
],
strategy: :one_for_one
)
end
end

@ -4,6 +4,7 @@ defmodule Indexer.Temporary.AddressesWithoutCode do
""" """
use GenServer use GenServer
use Indexer.Fetcher
require Logger require Logger

@ -1,38 +0,0 @@
defmodule Indexer.Temporary.AddressesWithoutCode.Supervisor do
@moduledoc """
Supervises `Indexer.Temporary.FailedCreatedAddresses`.
"""
use Supervisor
alias Indexer.Temporary.AddressesWithoutCode
def child_spec([init_arguments]) do
child_spec([init_arguments, []])
end
def child_spec([_init_arguments, _gen_server_options] = start_link_arguments) do
default = %{
id: __MODULE__,
start: {__MODULE__, :start_link, start_link_arguments},
type: :supervisor
}
Supervisor.child_spec(default, [])
end
def start_link(fetcher, gen_server_options \\ []) do
Supervisor.start_link(__MODULE__, fetcher, gen_server_options)
end
@impl Supervisor
def init(fetcher) do
Supervisor.init(
[
{Task.Supervisor, name: Indexer.Temporary.AddressesWithoutCode.TaskSupervisor},
{AddressesWithoutCode, [fetcher, [name: FailedCreatedAddresses]]}
],
strategy: :rest_for_one
)
end
end

@ -3,6 +3,7 @@ defmodule Indexer.Temporary.FailedCreatedAddresses do
Temporary module to fix internal transactions and their created transactions if a parent transaction has failed. Temporary module to fix internal transactions and their created transactions if a parent transaction has failed.
""" """
use GenServer use GenServer
use Indexer.Fetcher
require Logger require Logger
@ -10,6 +11,7 @@ defmodule Indexer.Temporary.FailedCreatedAddresses do
alias Explorer.Chain.{Address, Data, InternalTransaction, Transaction} alias Explorer.Chain.{Address, Data, InternalTransaction, Transaction}
alias Explorer.Repo alias Explorer.Repo
alias Indexer.Fetcher.ContractCode
alias Indexer.Temporary.FailedCreatedAddresses.TaskSupervisor alias Indexer.Temporary.FailedCreatedAddresses.TaskSupervisor
@task_options [max_concurrency: 3, timeout: :infinity] @task_options [max_concurrency: 3, timeout: :infinity]
@ -97,7 +99,7 @@ defmodule Indexer.Temporary.FailedCreatedAddresses do
:ok = :ok =
internal_transaction internal_transaction
|> code_entry() |> code_entry()
|> Indexer.Code.Fetcher.run(json_rpc_named_arguments) |> ContractCode.run(json_rpc_named_arguments)
end) end)
Logger.debug( Logger.debug(

@ -1,38 +0,0 @@
defmodule Indexer.Temporary.FailedCreatedAddresses.Supervisor do
@moduledoc """
Supervises `Indexer.Temporary.FailedCreatedAddresses`.
"""
use Supervisor
alias Indexer.Temporary.FailedCreatedAddresses
def child_spec([init_arguments]) do
child_spec([init_arguments, []])
end
def child_spec([_init_arguments, _gen_server_options] = start_link_arguments) do
default = %{
id: __MODULE__,
start: {__MODULE__, :start_link, start_link_arguments},
type: :supervisor
}
Supervisor.child_spec(default, [])
end
def start_link(json_rpc_named_arguments, gen_server_options \\ []) do
Supervisor.start_link(__MODULE__, json_rpc_named_arguments, gen_server_options)
end
@impl Supervisor
def init(json_rpc_named_arguments) do
Supervisor.init(
[
{Task.Supervisor, name: Indexer.Temporary.FailedCreatedAddresses.TaskSupervisor},
{FailedCreatedAddresses, [json_rpc_named_arguments, [name: FailedCreatedAddresses]]}
],
strategy: :rest_for_one
)
end
end

@ -1,4 +1,4 @@
defmodule Indexer.TokenTransfer.Uncataloged.Worker do defmodule Indexer.Temporary.UncatalogedTokenTransfers do
@moduledoc """ @moduledoc """
Catalogs token tranfer logs missing an accompanying token transfer record. Catalogs token tranfer logs missing an accompanying token transfer record.
@ -8,12 +8,13 @@ defmodule Indexer.TokenTransfer.Uncataloged.Worker do
""" """
use GenServer use GenServer
use Indexer.Fetcher
require Logger require Logger
alias Explorer.Chain alias Explorer.Chain
alias Indexer.Block.Catchup.Fetcher alias Indexer.Block.Catchup.Fetcher
alias Indexer.TokenTransfer.Uncataloged alias Indexer.Temporary.UncatalogedTokenTransfers
def child_spec([init_arguments]) do def child_spec([init_arguments]) do
child_spec([init_arguments, []]) child_spec([init_arguments, []])
@ -93,6 +94,6 @@ defmodule Indexer.TokenTransfer.Uncataloged.Worker do
end end
defp async_push_front(block_numbers) do defp async_push_front(block_numbers) do
Task.Supervisor.async_nolink(Uncataloged.TaskSupervisor, Fetcher, :push_front, [block_numbers]) Task.Supervisor.async_nolink(UncatalogedTokenTransfers.TaskSupervisor, Fetcher, :push_front, [block_numbers])
end end
end end

@ -1,41 +0,0 @@
defmodule Indexer.Token.Supervisor do
@moduledoc """
Supervises `Indexer.Token.Fetcher` and its batch tasks through `Indexer.Token.TaskSupervisor`.
"""
use Supervisor
alias Indexer.Token.{Fetcher, MetadataUpdater}
def child_spec([init_arguments]) do
child_spec([init_arguments, []])
end
def child_spec([_init_arguments, _gen_server_options] = start_link_arguments) do
default = %{
id: __MODULE__,
start: {__MODULE__, :start_link, start_link_arguments},
type: :supervisor
}
Supervisor.child_spec(default, [])
end
def start_link(arguments, gen_server_options \\ []) do
Supervisor.start_link(__MODULE__, arguments, Keyword.put_new(gen_server_options, :name, __MODULE__))
end
@impl Supervisor
def init(fetcher_arguments) do
metadata_updater_inverval = Application.get_env(:indexer, :metadata_updater_days_interval)
Supervisor.init(
[
{Task.Supervisor, name: Indexer.Token.TaskSupervisor},
{Fetcher, [fetcher_arguments, [name: Fetcher]]},
{MetadataUpdater, %{update_interval: metadata_updater_inverval}}
],
strategy: :one_for_one
)
end
end

@ -1,38 +0,0 @@
defmodule Indexer.TokenBalance.Supervisor do
@moduledoc """
Supervises `Indexer.TokenBalance.Fetcher` and its batch tasks through `Indexer.TokenBalance.TaskSupervisor`
"""
use Supervisor
alias Indexer.TokenBalance.Fetcher
def child_spec([init_arguments]) do
child_spec([init_arguments, []])
end
def child_spec([_init_arguments, _gen_server_options] = start_link_arguments) do
default = %{
id: __MODULE__,
start: {__MODULE__, :start_link, start_link_arguments},
type: :supervisor
}
Supervisor.child_spec(default, [])
end
def start_link(arguments, gen_server_options \\ []) do
Supervisor.start_link(__MODULE__, arguments, Keyword.put_new(gen_server_options, :name, __MODULE__))
end
@impl Supervisor
def init(fetcher_arguments) do
Supervisor.init(
[
{Task.Supervisor, name: Indexer.TokenBalance.TaskSupervisor},
{Fetcher, [fetcher_arguments, [name: Fetcher]]}
],
strategy: :one_for_one
)
end
end

@ -10,13 +10,14 @@ defmodule Indexer.TokenBalances do
alias Explorer.Chain alias Explorer.Chain
alias Explorer.Token.BalanceReader alias Explorer.Token.BalanceReader
alias Indexer.{TokenBalance, Tracer} alias Indexer.Fetcher.TokenBalance
alias Indexer.Tracer
@doc """ @doc """
Fetches TokenBalances from specific Addresses and Blocks in the Blockchain Fetches TokenBalances from specific Addresses and Blocks in the Blockchain
In case an exception is raised during the RPC call the particular TokenBalance request In case an exception is raised during the RPC call the particular TokenBalance request
is ignored and sent to `TokenBalance.Fetcher` to be fetched again. is ignored and sent to `TokenBalance` to be fetched again.
## token_balances ## token_balances
@ -84,7 +85,7 @@ defmodule Indexer.TokenBalances do
block_number: token_balance.block_number block_number: token_balance.block_number
}) })
end) end)
|> TokenBalance.Fetcher.async_fetch() |> TokenBalance.async_fetch()
end end
defp ignore_request_with_errors(%{value: nil, value_fetched_at: nil, error: _error}), do: false defp ignore_request_with_errors(%{value: nil, value_fetched_at: nil, error: _error}), do: false

@ -1,44 +0,0 @@
defmodule Indexer.TokenTransfer.Uncataloged.Supervisor do
@moduledoc """
Supervises process for ensuring uncataloged token transfers get queued for indexing.
"""
use Supervisor
alias Indexer.TokenTransfer.Uncataloged.Worker
def child_spec([]) do
child_spec([[]])
end
def child_spec([init_arguments]) do
child_spec([init_arguments, [name: __MODULE__]])
end
def child_spec([_init_arguments, _gen_server_options] = start_link_arguments) do
spec = %{
id: __MODULE__,
start: {__MODULE__, :start_link, start_link_arguments},
restart: :transient,
type: :supervisor
}
Supervisor.child_spec(spec, [])
end
def start_link(init_arguments, gen_server_options \\ []) do
Supervisor.start_link(__MODULE__, init_arguments, gen_server_options)
end
@impl Supervisor
def init(_) do
children = [
{Worker, [[supervisor: self()], [name: Worker]]},
{Task.Supervisor, name: Indexer.TokenTransfer.Uncataloged.TaskSupervisor}
]
opts = [strategy: :one_for_all]
Supervisor.init(children, opts)
end
end

@ -1,9 +0,0 @@
defmodule Indexer.TokenTransfers do
@moduledoc """
Context for working with token transfers.
"""
alias Indexer.TokenTransfer.Parser
defdelegate parse(items), to: Parser
end

@ -1,4 +1,4 @@
defmodule Indexer.Address.CoinBalances do defmodule Indexer.Transform.AddressCoinBalances do
@moduledoc """ @moduledoc """
Extracts `Explorer.Chain.Address.CoinBalance` params from other schema's params. Extracts `Explorer.Chain.Address.CoinBalance` params from other schema's params.
""" """

@ -1,4 +1,4 @@
defmodule Indexer.Address.TokenBalances do defmodule Indexer.Transform.AddressTokenBalances do
@moduledoc """ @moduledoc """
Extracts `Explorer.Address.TokenBalance` params from other schema's params. Extracts `Explorer.Address.TokenBalance` params from other schema's params.
""" """

@ -1,4 +1,4 @@
defmodule Indexer.AddressExtraction do defmodule Indexer.Transform.Addresses do
@moduledoc """ @moduledoc """
Extract Addresses from data fetched from the Blockchain and structured as Blocks, InternalTransactions, Extract Addresses from data fetched from the Blockchain and structured as Blocks, InternalTransactions,
Transactions and Logs. Transactions and Logs.
@ -154,7 +154,7 @@ defmodule Indexer.AddressExtraction do
Blocks have their `miner_hash` extracted. Blocks have their `miner_hash` extracted.
iex> Indexer.AddressExtraction.extract_addresses( iex> Indexer.Addresses.extract_addresses(
...> %{ ...> %{
...> blocks: [ ...> blocks: [
...> %{ ...> %{
@ -174,7 +174,7 @@ defmodule Indexer.AddressExtraction do
Internal transactions can have their `from_address_hash`, `to_address_hash` and/or `created_contract_address_hash` Internal transactions can have their `from_address_hash`, `to_address_hash` and/or `created_contract_address_hash`
extracted. extracted.
iex> Indexer.AddressExtraction.extract_addresses( iex> Indexer.Addresses.extract_addresses(
...> %{ ...> %{
...> internal_transactions: [ ...> internal_transactions: [
...> %{ ...> %{
@ -211,7 +211,7 @@ defmodule Indexer.AddressExtraction do
Transactions can have their `from_address_hash` and/or `to_address_hash` extracted. Transactions can have their `from_address_hash` and/or `to_address_hash` extracted.
iex> Indexer.AddressExtraction.extract_addresses( iex> Indexer.Addresses.extract_addresses(
...> %{ ...> %{
...> transactions: [ ...> transactions: [
...> %{ ...> %{
@ -247,7 +247,7 @@ defmodule Indexer.AddressExtraction do
Logs can have their `address_hash` extracted. Logs can have their `address_hash` extracted.
iex> Indexer.AddressExtraction.extract_addresses( iex> Indexer.Addresses.extract_addresses(
...> %{ ...> %{
...> logs: [ ...> logs: [
...> %{ ...> %{
@ -266,7 +266,7 @@ defmodule Indexer.AddressExtraction do
When the same address is mentioned multiple times, the greatest `block_number` is used When the same address is mentioned multiple times, the greatest `block_number` is used
iex> Indexer.AddressExtraction.extract_addresses( iex> Indexer.Addresses.extract_addresses(
...> %{ ...> %{
...> blocks: [ ...> blocks: [
...> %{ ...> %{
@ -319,7 +319,7 @@ defmodule Indexer.AddressExtraction do
When a contract is created and then used in internal transactions and transaction in the same fetched data, the When a contract is created and then used in internal transactions and transaction in the same fetched data, the
`created_contract_code` is merged with the greatest `block_number` `created_contract_code` is merged with the greatest `block_number`
iex> Indexer.AddressExtraction.extract_addresses( iex> Indexer.Addresses.extract_addresses(
...> %{ ...> %{
...> internal_transactions: [ ...> internal_transactions: [
...> %{ ...> %{

@ -1,8 +1,24 @@
defmodule Indexer.Block.Util do defmodule Indexer.Transform.Blocks do
@moduledoc """ @moduledoc """
Helper functions for parsing block information. Protocol for transforming blocks.
""" """
@type block :: map()
@doc """
Transforms a block.
"""
@callback transform(block :: block()) :: block()
@doc """
Runs a list of blocks through the configured block transformer.
"""
def transform_blocks(blocks) when is_list(blocks) do
transformer = Application.get_env(:indexer, :block_transformer)
Enum.map(blocks, &transformer.transform/1)
end
@doc """ @doc """
Calculates the signer's address by recovering the ECDSA public key. Calculates the signer's address by recovering the ECDSA public key.

@ -1,13 +1,13 @@
defmodule Indexer.Block.Transform.Base do defmodule Indexer.Transform.Blocks.Base do
@moduledoc """ @moduledoc """
Default block transformer to be used. Default block transformer to be used.
""" """
alias Indexer.Block.Transform alias Indexer.Transform.Blocks
@behaviour Transform @behaviour Blocks
@impl Transform @impl Blocks
def transform(block) when is_map(block) do def transform(block) when is_map(block) do
block block
end end

@ -1,17 +1,17 @@
defmodule Indexer.Block.Transform.Clique do defmodule Indexer.Transform.Blocks.Clique do
@moduledoc """ @moduledoc """
Handles block transforms for Clique chain. Handles block transforms for Clique chain.
""" """
alias Indexer.Block.{Transform, Util} alias Indexer.Transform.Blocks
@behaviour Transform @behaviour Blocks
@impl Transform @impl Blocks
def transform(%{number: 0} = block), do: block def transform(%{number: 0} = block), do: block
def transform(block) when is_map(block) do def transform(block) when is_map(block) do
miner_address = Util.signer(block) miner_address = Blocks.signer(block)
%{block | miner_hash: miner_address} %{block | miner_hash: miner_address}
end end

@ -1,4 +1,4 @@
defmodule Indexer.MintTransfer do defmodule Indexer.Transform.MintTransfers do
@moduledoc """ @moduledoc """
Helper functions to parse addresses from mint transfers. Helper functions to parse addresses from mint transfers.
@ -14,7 +14,7 @@ defmodule Indexer.MintTransfer do
## Examples ## Examples
iex> Indexer.MintTransfer.parse([ iex> Indexer.Transform.MintTransfers.parse([
...> %{ ...> %{
...> address_hash: "0x867305d19606aadba405ce534e303d0e225f9556", ...> address_hash: "0x867305d19606aadba405ce534e303d0e225f9556",
...> block_number: 137_194, ...> block_number: 137_194,

@ -1,4 +1,4 @@
defmodule Indexer.TokenTransfer.Parser do defmodule Indexer.Transform.TokenTransfers do
@moduledoc """ @moduledoc """
Helper functions for transforming data for ERC-20 and ERC-721 token transfers. Helper functions for transforming data for ERC-20 and ERC-721 token transfers.
""" """

@ -7,8 +7,18 @@ defmodule Indexer.Block.Catchup.BoundIntervalSupervisorTest do
import EthereumJSONRPC, only: [integer_to_quantity: 1] import EthereumJSONRPC, only: [integer_to_quantity: 1]
alias Explorer.Chain.Block alias Explorer.Chain.Block
alias Indexer.{BoundInterval, Code, CoinBalance, InternalTransaction, ReplacedTransaction, Token, TokenBalance} alias Indexer.BoundInterval
alias Indexer.Block.{Catchup, Uncle} alias Indexer.Block.Catchup
alias Indexer.Fetcher.{
CoinBalance,
ContractCode,
InternalTransaction,
ReplacedTransaction,
Token,
TokenBalance,
UncleBlock
}
@moduletag capture_log: true @moduletag capture_log: true
@ -204,12 +214,12 @@ defmodule Indexer.Block.Catchup.BoundIntervalSupervisorTest do
CoinBalance.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) CoinBalance.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments)
InternalTransaction.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) InternalTransaction.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments)
Code.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) ContractCode.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments)
Token.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) Token.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments)
TokenBalance.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) TokenBalance.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments)
ReplacedTransaction.Supervisor.Case.start_supervised!() ReplacedTransaction.Supervisor.Case.start_supervised!()
Uncle.Supervisor.Case.start_supervised!( UncleBlock.Supervisor.Case.start_supervised!(
block_fetcher: %Indexer.Block.Fetcher{json_rpc_named_arguments: json_rpc_named_arguments} block_fetcher: %Indexer.Block.Fetcher{json_rpc_named_arguments: json_rpc_named_arguments}
) )
@ -410,7 +420,7 @@ defmodule Indexer.Block.Catchup.BoundIntervalSupervisorTest do
start_supervised!({Task.Supervisor, name: Indexer.Block.Catchup.TaskSupervisor}) start_supervised!({Task.Supervisor, name: Indexer.Block.Catchup.TaskSupervisor})
CoinBalance.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) CoinBalance.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments)
Code.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) ContractCode.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments)
InternalTransaction.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) InternalTransaction.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments)
Token.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) Token.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments)
TokenBalance.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) TokenBalance.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments)
@ -504,12 +514,12 @@ defmodule Indexer.Block.Catchup.BoundIntervalSupervisorTest do
start_supervised({Task.Supervisor, name: Indexer.Block.Catchup.TaskSupervisor}) start_supervised({Task.Supervisor, name: Indexer.Block.Catchup.TaskSupervisor})
CoinBalance.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) CoinBalance.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments)
InternalTransaction.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) InternalTransaction.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments)
Code.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) ContractCode.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments)
Token.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) Token.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments)
TokenBalance.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) TokenBalance.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments)
ReplacedTransaction.Supervisor.Case.start_supervised!() ReplacedTransaction.Supervisor.Case.start_supervised!()
Uncle.Supervisor.Case.start_supervised!( UncleBlock.Supervisor.Case.start_supervised!(
block_fetcher: %Indexer.Block.Fetcher{json_rpc_named_arguments: json_rpc_named_arguments} block_fetcher: %Indexer.Block.Fetcher{json_rpc_named_arguments: json_rpc_named_arguments}
) )

@ -7,8 +7,9 @@ defmodule Indexer.Block.Catchup.FetcherTest do
alias Explorer.Chain alias Explorer.Chain
alias Explorer.Chain.Block.Reward alias Explorer.Chain.Block.Reward
alias Indexer.{Block, CoinBalance, InternalTransaction, Token, TokenBalance} alias Indexer.Block
alias Indexer.Block.Catchup.Fetcher alias Indexer.Block.Catchup.Fetcher
alias Indexer.Fetcher.{BlockReward, CoinBalance, InternalTransaction, Token, TokenBalance, UncleBlock}
@moduletag capture_log: true @moduletag capture_log: true
@ -48,7 +49,7 @@ defmodule Indexer.Block.Catchup.FetcherTest do
end end
end) end)
Process.register(pid, Block.Uncle.Fetcher) Process.register(pid, UncleBlock)
nephew_hash = block_hash() |> to_string() nephew_hash = block_hash() |> to_string()
uncle_hash = block_hash() |> to_string() uncle_hash = block_hash() |> to_string()
@ -315,7 +316,7 @@ defmodule Indexer.Block.Catchup.FetcherTest do
end end
end) end)
Process.register(pid, Indexer.Block.Reward.Fetcher) Process.register(pid, BlockReward)
assert %{first_block_number: ^block_number, missing_block_count: 1, shrunk: false} = assert %{first_block_number: ^block_number, missing_block_count: 1, shrunk: false} =
Fetcher.task(%Fetcher{ Fetcher.task(%Fetcher{
@ -414,7 +415,7 @@ defmodule Indexer.Block.Catchup.FetcherTest do
end end
end) end)
Process.register(pid, Indexer.Block.Reward.Fetcher) Process.register(pid, BlockReward)
assert %{first_block_number: ^block_number, missing_block_count: 1, shrunk: false} = assert %{first_block_number: ^block_number, missing_block_count: 1, shrunk: false} =
Fetcher.task(%Fetcher{ Fetcher.task(%Fetcher{

@ -1,8 +1,8 @@
defmodule Indexer.SequenceTest do defmodule Indexer.Block.Catchup.SequenceTest do
use ExUnit.Case use ExUnit.Case
alias Indexer.Block.Catchup.Sequence
alias Indexer.Memory.Shrinkable alias Indexer.Memory.Shrinkable
alias Indexer.Sequence
describe "start_link/1" do describe "start_link/1" do
test "without :ranges with :first with positive step pops infinitely" do test "without :ranges with :first with positive step pops infinitely" do

@ -9,8 +9,18 @@ defmodule Indexer.Block.FetcherTest do
alias Explorer.Chain alias Explorer.Chain
alias Explorer.Chain.{Address, Log, Transaction, Wei} alias Explorer.Chain.{Address, Log, Transaction, Wei}
alias Indexer.{CoinBalance, BufferedTask, Code, InternalTransaction, ReplacedTransaction, Token, TokenBalance} alias Indexer.Block.Fetcher
alias Indexer.Block.{Fetcher, Uncle} alias Indexer.BufferedTask
alias Indexer.Fetcher.{
CoinBalance,
ContractCode,
InternalTransaction,
ReplacedTransaction,
Token,
TokenBalance,
UncleBlock
}
@moduletag capture_log: true @moduletag capture_log: true
@ -41,13 +51,13 @@ defmodule Indexer.Block.FetcherTest do
describe "import_range/2" do describe "import_range/2" do
setup %{json_rpc_named_arguments: json_rpc_named_arguments} do setup %{json_rpc_named_arguments: json_rpc_named_arguments} do
CoinBalance.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) CoinBalance.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments)
Code.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) ContractCode.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments)
InternalTransaction.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) InternalTransaction.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments)
Token.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) Token.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments)
TokenBalance.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) TokenBalance.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments)
ReplacedTransaction.Supervisor.Case.start_supervised!() ReplacedTransaction.Supervisor.Case.start_supervised!()
Uncle.Supervisor.Case.start_supervised!( UncleBlock.Supervisor.Case.start_supervised!(
block_fetcher: %Fetcher{json_rpc_named_arguments: json_rpc_named_arguments} block_fetcher: %Fetcher{json_rpc_named_arguments: json_rpc_named_arguments}
) )
@ -226,8 +236,8 @@ defmodule Indexer.Block.FetcherTest do
errors: [] errors: []
}} = result }} = result
wait_for_tasks(InternalTransaction.Fetcher) wait_for_tasks(InternalTransaction)
wait_for_tasks(CoinBalance.Fetcher) wait_for_tasks(CoinBalance)
assert Repo.aggregate(Chain.Block, :count, :hash) == 1 assert Repo.aggregate(Chain.Block, :count, :hash) == 1
assert Repo.aggregate(Address, :count, :hash) == 1 assert Repo.aggregate(Address, :count, :hash) == 1
@ -492,8 +502,8 @@ defmodule Indexer.Block.FetcherTest do
] ]
}} = Fetcher.fetch_and_import_range(block_fetcher, block_number..block_number) }} = Fetcher.fetch_and_import_range(block_fetcher, block_number..block_number)
wait_for_tasks(InternalTransaction.Fetcher) wait_for_tasks(InternalTransaction)
wait_for_tasks(CoinBalance.Fetcher) wait_for_tasks(CoinBalance)
assert Repo.aggregate(Block, :count, :hash) == 1 assert Repo.aggregate(Block, :count, :hash) == 1
assert Repo.aggregate(Address, :count, :hash) == 5 assert Repo.aggregate(Address, :count, :hash) == 5
@ -587,8 +597,8 @@ defmodule Indexer.Block.FetcherTest do
errors: [] errors: []
}} = Fetcher.fetch_and_import_range(block_fetcher, block_number..block_number) }} = Fetcher.fetch_and_import_range(block_fetcher, block_number..block_number)
wait_for_tasks(InternalTransaction.Fetcher) wait_for_tasks(InternalTransaction)
wait_for_tasks(CoinBalance.Fetcher) wait_for_tasks(CoinBalance)
assert Repo.aggregate(Chain.Block, :count, :hash) == 1 assert Repo.aggregate(Chain.Block, :count, :hash) == 1
assert Repo.aggregate(Address, :count, :hash) == 2 assert Repo.aggregate(Address, :count, :hash) == 2

@ -6,8 +6,9 @@ defmodule Indexer.Block.Realtime.FetcherTest do
alias Explorer.Chain alias Explorer.Chain
alias Explorer.Chain.{Address, Transaction} alias Explorer.Chain.{Address, Transaction}
alias Indexer.{Sequence, Token, TokenBalance, ReplacedTransaction} alias Indexer.Block.Catchup.Sequence
alias Indexer.Block.{Realtime, Uncle} alias Indexer.Block.Realtime
alias Indexer.Fetcher.{ReplacedTransaction, Token, TokenBalance, UncleBlock}
@moduletag capture_log: true @moduletag capture_log: true
@ -50,7 +51,7 @@ defmodule Indexer.Block.Realtime.FetcherTest do
Token.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) Token.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments)
Uncle.Supervisor.Case.start_supervised!( UncleBlock.Supervisor.Case.start_supervised!(
block_fetcher: %Indexer.Block.Fetcher{json_rpc_named_arguments: json_rpc_named_arguments} block_fetcher: %Indexer.Block.Fetcher{json_rpc_named_arguments: json_rpc_named_arguments}
) )

@ -1,40 +0,0 @@
defmodule Indexer.Block.UtilTest do
use ExUnit.Case
alias Indexer.Block.Util
test "signer/1" do
data = %{
difficulty: 1,
extra_data:
"0xd68301080d846765746886676f312e3130856c696e7578000000000000000000773ab2ca8f47904a14739ad80a75b71d9d29b9fff8b7ecdcb73efffa6f74122f17d304b5dc8e6e5f256c9474dd115c8d4dae31b7a3d409e5c3270f8fde41cd8c00",
gas_limit: 7_753_377,
gas_used: 1_810_195,
hash: "0x7004c895e812c55b0c2be8a46d72ca300a683dc27d1d7917ee7742d4d0359c1f",
logs_bloom:
"0x00000000000000020000000000002000000400000000000000000000000000000000000000000000040000080004000020000010000000000000000000000000000000000000000008000008000000000000000000200000000000000000000000000000020000000000000000000800000000000000804000000010080000000800000000000000000000000000000000000000000000800000000000080000000008000400000000404000000000000000000000000200000000000000000000000002000000000000001002000000000000002000000008000000000020000000000000000000000000000000000000000000000000400000800000000000",
miner_hash: "0x0000000000000000000000000000000000000000",
mix_hash: "0x0000000000000000000000000000000000000000000000000000000000000000",
nonce: "0x0000000000000000",
number: 2_848_394,
parent_hash: "0x20350fc367e19d3865be1ea7da72ab81f8f9941c43ac6bb24a34a0a7caa2f3df",
receipts_root: "0x6ade4ac1079ea50cfadcce2b75ffbe4f9b14bf69b4607bbf1739463076ca6246",
sha3_uncles: "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347",
size: 6437,
state_root: "0x23f63347851bcd109059d007d71e19c4f5e73b7f0862bebcd04458333a004d92",
timestamp: DateTime.from_unix!(1_534_796_040),
total_difficulty: 5_353_647,
transactions: [
"0x7e3bb851fc74a436826d2af6b96e4db9484431811ef0d9c9e78370488d33d4e5",
"0x3976fd1e3d2a715c3cfcfde9bd3210798c26c017b8edb841d319227ecb3322fb",
"0xd8db124005bb8b6fda7b71fd56ac782552a66af58fe843ba3c4930423b87d1d2",
"0x10c1a1ca4d9f4b2bd5b89f7bbcbbc2d69e166fe23662b8db4f6beae0f50ac9fd",
"0xaa58a6545677c796a56b8bc874174c8cfd31a6c6e6ca3a87e086d4f66d52858a"
],
transactions_root: "0xde8d25c0b9b54310128a21601331094b43f910f9f96102869c2e2dca94884bf4",
uncles: []
}
assert Util.signer(data) == "0xfc18cbc391de84dbd87db83b20935d3e89f5dd91"
end
end

@ -1,4 +1,4 @@
defmodule Indexer.Block.Reward.FetcherTest do defmodule Indexer.Fetcher.BlockRewardTest do
# MUST be `async: false` so that {:shared, pid} is set for connection to allow CoinBalanceFetcher's self-send to have # MUST be `async: false` so that {:shared, pid} is set for connection to allow CoinBalanceFetcher's self-send to have
# connection allowed immediately. # connection allowed immediately.
use EthereumJSONRPC.Case, async: false use EthereumJSONRPC.Case, async: false
@ -9,8 +9,8 @@ defmodule Indexer.Block.Reward.FetcherTest do
alias Explorer.Chain alias Explorer.Chain
alias Explorer.Chain.{Block, Hash, Wei} alias Explorer.Chain.{Block, Hash, Wei}
alias Indexer.Block.Reward
alias Indexer.BufferedTask alias Indexer.BufferedTask
alias Indexer.Fetcher.BlockReward
@moduletag :capture_log @moduletag :capture_log
@ -37,32 +37,32 @@ defmodule Indexer.Block.Reward.FetcherTest do
describe "init/3" do describe "init/3" do
test "without blocks" do test "without blocks" do
assert [] = Reward.Fetcher.init([], &[&1 | &2], nil) assert [] = BlockReward.init([], &[&1 | &2], nil)
end end
test "with consensus block without reward" do test "with consensus block without reward" do
%Block{number: block_number} = insert(:block) %Block{number: block_number} = insert(:block)
assert [^block_number] = Reward.Fetcher.init([], &[&1 | &2], nil) assert [^block_number] = BlockReward.init([], &[&1 | &2], nil)
end end
test "with consensus block with reward" do test "with consensus block with reward" do
block = insert(:block) block = insert(:block)
insert(:reward, address_hash: block.miner_hash, block_hash: block.hash) insert(:reward, address_hash: block.miner_hash, block_hash: block.hash)
assert [] = Reward.Fetcher.init([], &[&1 | &2], nil) assert [] = BlockReward.init([], &[&1 | &2], nil)
end end
test "with non-consensus block" do test "with non-consensus block" do
insert(:block, consensus: false) insert(:block, consensus: false)
assert [] = Reward.Fetcher.init([], &[&1 | &2], nil) assert [] = BlockReward.init([], &[&1 | &2], nil)
end end
end end
describe "async_fetch/1" do describe "async_fetch/1" do
setup %{json_rpc_named_arguments: json_rpc_named_arguments} do setup %{json_rpc_named_arguments: json_rpc_named_arguments} do
Reward.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) BlockReward.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments)
block = insert(:block) block = insert(:block)
@ -130,11 +130,11 @@ defmodule Indexer.Block.Reward.FetcherTest do
end end
end) end)
Process.register(pid, Indexer.CoinBalance.Fetcher) Process.register(pid, Indexer.Fetcher.CoinBalance)
assert :ok = Reward.Fetcher.async_fetch([block_number]) assert :ok = BlockReward.async_fetch([block_number])
wait_for_tasks(Reward.Fetcher) wait_for_tasks(BlockReward)
assert count(Chain.Block.Reward) == 1 assert count(Chain.Block.Reward) == 1
assert_receive {:balance_fields, [{^miner_hash_bytes, ^block_number}]}, 500 assert_receive {:balance_fields, [{^miner_hash_bytes, ^block_number}]}, 500
@ -201,11 +201,11 @@ defmodule Indexer.Block.Reward.FetcherTest do
end end
end) end)
Process.register(pid, Indexer.CoinBalance.Fetcher) Process.register(pid, Indexer.Fetcher.CoinBalance)
assert :ok = Reward.Fetcher.async_fetch([block_number]) assert :ok = BlockReward.async_fetch([block_number])
wait_for_tasks(Reward.Fetcher) wait_for_tasks(BlockReward)
assert count(Chain.Block.Reward) == 1 assert count(Chain.Block.Reward) == 1
assert_receive {:balance_fields, [{^miner_hash_bytes, ^block_number}]}, 500 assert_receive {:balance_fields, [{^miner_hash_bytes, ^block_number}]}, 500
@ -256,9 +256,9 @@ defmodule Indexer.Block.Reward.FetcherTest do
} }
end) end)
assert :ok = Reward.Fetcher.async_fetch([block_number]) assert :ok = BlockReward.async_fetch([block_number])
wait_for_tasks(Reward.Fetcher) wait_for_tasks(BlockReward)
assert count(Chain.Block.Reward) == 0 assert count(Chain.Block.Reward) == 0
end end
@ -334,9 +334,9 @@ defmodule Indexer.Block.Reward.FetcherTest do
end end
end) end)
Process.register(pid, Indexer.CoinBalance.Fetcher) Process.register(pid, Indexer.Fetcher.CoinBalance)
assert :ok = Reward.Fetcher.run([block_number], json_rpc_named_arguments) assert :ok = BlockReward.run([block_number], json_rpc_named_arguments)
assert count(Chain.Block.Reward) == 1 assert count(Chain.Block.Reward) == 1
assert count(Chain.Address.CoinBalance) == 1 assert count(Chain.Address.CoinBalance) == 1
@ -422,9 +422,9 @@ defmodule Indexer.Block.Reward.FetcherTest do
end end
end) end)
Process.register(pid, Indexer.CoinBalance.Fetcher) Process.register(pid, Indexer.Fetcher.CoinBalance)
assert :ok = Reward.Fetcher.run([block_number], json_rpc_named_arguments) assert :ok = BlockReward.run([block_number], json_rpc_named_arguments)
assert count(Chain.Block.Reward) == 2 assert count(Chain.Block.Reward) == 2
assert count(Chain.Address.CoinBalance) == 2 assert count(Chain.Address.CoinBalance) == 2
@ -504,9 +504,9 @@ defmodule Indexer.Block.Reward.FetcherTest do
end end
end) end)
Process.register(pid, Indexer.CoinBalance.Fetcher) Process.register(pid, Indexer.Fetcher.CoinBalance)
assert :ok = Reward.Fetcher.run([block_number], json_rpc_named_arguments) assert :ok = BlockReward.run([block_number], json_rpc_named_arguments)
assert count(Chain.Block.Reward) == 1 assert count(Chain.Block.Reward) == 1
assert count(Chain.Address.CoinBalance) == 1 assert count(Chain.Address.CoinBalance) == 1
@ -565,7 +565,7 @@ defmodule Indexer.Block.Reward.FetcherTest do
} }
end) end)
assert :ok = Reward.Fetcher.run([block_number], json_rpc_named_arguments) assert :ok = BlockReward.run([block_number], json_rpc_named_arguments)
assert count(Chain.Block.Reward) == 0 assert count(Chain.Block.Reward) == 0
assert count(Chain.Address.CoinBalance) == 0 assert count(Chain.Address.CoinBalance) == 0
@ -639,10 +639,10 @@ defmodule Indexer.Block.Reward.FetcherTest do
end end
end) end)
Process.register(pid, Indexer.CoinBalance.Fetcher) Process.register(pid, Indexer.Fetcher.CoinBalance)
assert {:retry, [^error_block_number]} = assert {:retry, [^error_block_number]} =
Reward.Fetcher.run([block_number, error_block_number], json_rpc_named_arguments) BlockReward.run([block_number, error_block_number], json_rpc_named_arguments)
assert count(Chain.Block.Reward) == 1 assert count(Chain.Block.Reward) == 1
assert count(Chain.Address.CoinBalance) == 1 assert count(Chain.Address.CoinBalance) == 1

@ -1,4 +1,4 @@
defmodule Indexer.CoinBalance.OnDemandFetcherTest do defmodule Indexer.Fetcher.CoinBalanceOnDemandTest do
# MUST be `async: false` so that {:shared, pid} is set for connection to allow CoinBalanceFetcher's self-send to have # MUST be `async: false` so that {:shared, pid} is set for connection to allow CoinBalanceFetcher's self-send to have
# connection allowed immediately. # connection allowed immediately.
use EthereumJSONRPC.Case, async: false use EthereumJSONRPC.Case, async: false
@ -9,7 +9,7 @@ defmodule Indexer.CoinBalance.OnDemandFetcherTest do
alias Explorer.Chain.Events.Subscriber alias Explorer.Chain.Events.Subscriber
alias Explorer.Chain.Wei alias Explorer.Chain.Wei
alias Explorer.Counters.AverageBlockTime alias Explorer.Counters.AverageBlockTime
alias Indexer.CoinBalance.OnDemandFetcher alias Indexer.Fetcher.CoinBalanceOnDemand
@moduletag :capture_log @moduletag :capture_log
@ -24,7 +24,7 @@ defmodule Indexer.CoinBalance.OnDemandFetcherTest do
start_supervised!({Task.Supervisor, name: Indexer.TaskSupervisor}) start_supervised!({Task.Supervisor, name: Indexer.TaskSupervisor})
start_supervised!(AverageBlockTime) start_supervised!(AverageBlockTime)
start_supervised!({OnDemandFetcher, [mocked_json_rpc_named_arguments, [name: OnDemandFetcher]]}) start_supervised!({CoinBalanceOnDemand, [mocked_json_rpc_named_arguments, [name: CoinBalanceOnDemand]]})
Application.put_env(:explorer, AverageBlockTime, enabled: true) Application.put_env(:explorer, AverageBlockTime, enabled: true)
@ -57,25 +57,25 @@ defmodule Indexer.CoinBalance.OnDemandFetcherTest do
test "treats all addresses as current if the average block time is disabled", %{stale_address: address} do test "treats all addresses as current if the average block time is disabled", %{stale_address: address} do
Application.put_env(:explorer, AverageBlockTime, enabled: false) Application.put_env(:explorer, AverageBlockTime, enabled: false)
assert OnDemandFetcher.trigger_fetch(address) == :current assert CoinBalanceOnDemand.trigger_fetch(address) == :current
end end
test "if the address has not been fetched within the last 24 hours of blocks it is considered stale", %{ test "if the address has not been fetched within the last 24 hours of blocks it is considered stale", %{
stale_address: address stale_address: address
} do } do
assert OnDemandFetcher.trigger_fetch(address) == {:stale, 1} assert CoinBalanceOnDemand.trigger_fetch(address) == {:stale, 1}
end end
test "if the address has been fetched within the last 24 hours of blocks it is considered current", %{ test "if the address has been fetched within the last 24 hours of blocks it is considered current", %{
current_address: address current_address: address
} do } do
assert OnDemandFetcher.trigger_fetch(address) == :current assert CoinBalanceOnDemand.trigger_fetch(address) == :current
end end
test "if there is an unfetched balance within the window for an address, it is considered pending", %{ test "if there is an unfetched balance within the window for an address, it is considered pending", %{
pending_address: pending_address pending_address: pending_address
} do } do
assert OnDemandFetcher.trigger_fetch(pending_address) == {:pending, 2} assert CoinBalanceOnDemand.trigger_fetch(pending_address) == {:pending, 2}
end end
end end
@ -111,7 +111,7 @@ defmodule Indexer.CoinBalance.OnDemandFetcherTest do
{:ok, [%{id: id, jsonrpc: "2.0", result: "0x02"}]} {:ok, [%{id: id, jsonrpc: "2.0", result: "0x02"}]}
end) end)
assert OnDemandFetcher.trigger_fetch(address) == {:stale, 1} assert CoinBalanceOnDemand.trigger_fetch(address) == {:stale, 1}
{:ok, expected_wei} = Wei.cast(2) {:ok, expected_wei} = Wei.cast(2)
@ -138,7 +138,7 @@ defmodule Indexer.CoinBalance.OnDemandFetcherTest do
{:ok, [%{id: id, jsonrpc: "2.0", result: "0x02"}]} {:ok, [%{id: id, jsonrpc: "2.0", result: "0x02"}]}
end) end)
assert OnDemandFetcher.trigger_fetch(address) == {:pending, 2} assert CoinBalanceOnDemand.trigger_fetch(address) == {:pending, 2}
{:ok, expected_wei} = Wei.cast(2) {:ok, expected_wei} = Wei.cast(2)

@ -1,4 +1,4 @@
defmodule Indexer.CoinBalance.FetcherTest do defmodule Indexer.Fetcher.CoinBalanceTest do
# MUST be `async: false` so that {:shared, pid} is set for connection to allow CoinBalanceFetcher's self-send to have # MUST be `async: false` so that {:shared, pid} is set for connection to allow CoinBalanceFetcher's self-send to have
# connection allowed immediately. # connection allowed immediately.
use EthereumJSONRPC.Case, async: false use EthereumJSONRPC.Case, async: false
@ -8,7 +8,7 @@ defmodule Indexer.CoinBalance.FetcherTest do
import Mox import Mox
alias Explorer.Chain.{Address, Hash, Wei} alias Explorer.Chain.{Address, Hash, Wei}
alias Indexer.CoinBalance alias Indexer.Fetcher.CoinBalance
@moduletag :capture_log @moduletag :capture_log
@ -182,7 +182,7 @@ defmodule Indexer.CoinBalance.FetcherTest do
CoinBalance.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) CoinBalance.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments)
assert :ok = CoinBalance.Fetcher.async_fetch_balances([%{address_hash: hash, block_number: block_number}]) assert :ok = CoinBalance.async_fetch_balances([%{address_hash: hash, block_number: block_number}])
address = address =
wait(fn -> wait(fn ->
@ -254,7 +254,7 @@ defmodule Indexer.CoinBalance.FetcherTest do
{:ok, %Hash{bytes: address_hash_bytes}} = Hash.Address.cast(hash_data) {:ok, %Hash{bytes: address_hash_bytes}} = Hash.Address.cast(hash_data)
entries = Enum.map(block_quantities, &{address_hash_bytes, quantity_to_integer(&1)}) entries = Enum.map(block_quantities, &{address_hash_bytes, quantity_to_integer(&1)})
case CoinBalance.Fetcher.run(entries, json_rpc_named_arguments) do case CoinBalance.run(entries, json_rpc_named_arguments) do
:ok -> :ok ->
balances = Repo.all(from(balance in Address.CoinBalance, where: balance.address_hash == ^hash_data)) balances = Repo.all(from(balance in Address.CoinBalance, where: balance.address_hash == ^hash_data))
@ -309,7 +309,7 @@ defmodule Indexer.CoinBalance.FetcherTest do
{:ok, [%{id: id, error: %{code: 1, message: "Bad"}}]} {:ok, [%{id: id, error: %{code: 1, message: "Bad"}}]}
end) end)
assert {:retry, ^entries} = CoinBalance.Fetcher.run(entries, json_rpc_named_arguments) assert {:retry, ^entries} = CoinBalance.run(entries, json_rpc_named_arguments)
end end
test "retries none if all imported and no fetch errors", %{json_rpc_named_arguments: json_rpc_named_arguments} do test "retries none if all imported and no fetch errors", %{json_rpc_named_arguments: json_rpc_named_arguments} do
@ -320,7 +320,7 @@ defmodule Indexer.CoinBalance.FetcherTest do
{:ok, [%{id: id, result: "0x1"}]} {:ok, [%{id: id, result: "0x1"}]}
end) end)
assert :ok = CoinBalance.Fetcher.run(entries, json_rpc_named_arguments) assert :ok = CoinBalance.run(entries, json_rpc_named_arguments)
end end
test "retries retries fetch errors if all imported", %{json_rpc_named_arguments: json_rpc_named_arguments} do test "retries retries fetch errors if all imported", %{json_rpc_named_arguments: json_rpc_named_arguments} do
@ -360,7 +360,7 @@ defmodule Indexer.CoinBalance.FetcherTest do
end) end)
assert {:retry, [{^address_hash_bytes, ^bad_block_number}]} = assert {:retry, [{^address_hash_bytes, ^bad_block_number}]} =
CoinBalance.Fetcher.run( CoinBalance.run(
[{address_hash_bytes, good_block_number}, {address_hash_bytes, bad_block_number}], [{address_hash_bytes, good_block_number}, {address_hash_bytes, bad_block_number}],
json_rpc_named_arguments json_rpc_named_arguments
) )

@ -1,4 +1,4 @@
defmodule Indexer.Code.FetcherTest do defmodule Indexer.Fetcher.ContractCodeTest do
use EthereumJSONRPC.Case, async: false use EthereumJSONRPC.Case, async: false
use Explorer.DataCase use Explorer.DataCase
@ -7,7 +7,7 @@ defmodule Indexer.Code.FetcherTest do
import Mox import Mox
alias Explorer.Chain.{Address, Transaction} alias Explorer.Chain.{Address, Transaction}
alias Indexer.Code alias Indexer.Fetcher.ContractCode
@moduletag :capture_log @moduletag :capture_log
@ -83,10 +83,10 @@ defmodule Indexer.Code.FetcherTest do
insert(:address, hash: address) insert(:address, hash: address)
insert(:transaction, hash: hash, created_contract_address_hash: address) insert(:transaction, hash: hash, created_contract_address_hash: address)
Code.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) ContractCode.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments)
assert :ok = assert :ok =
Code.Fetcher.async_fetch([ ContractCode.async_fetch([
%{created_contract_address_hash: address, block_number: block_number, hash: hash} %{created_contract_address_hash: address, block_number: block_number, hash: hash}
]) ])

@ -1,4 +1,4 @@
defmodule Indexer.InternalTransaction.FetcherTest do defmodule Indexer.Fetcher.InternalTransactionTest do
use EthereumJSONRPC.Case, async: false use EthereumJSONRPC.Case, async: false
use Explorer.DataCase use Explorer.DataCase
@ -6,8 +6,7 @@ defmodule Indexer.InternalTransaction.FetcherTest do
import Mox import Mox
alias Explorer.Chain.{Address, Hash, Transaction} alias Explorer.Chain.{Address, Hash, Transaction}
alias Indexer.Fetcher.{CoinBalance, InternalTransaction, PendingTransaction}
alias Indexer.{CoinBalance, InternalTransaction, PendingTransaction}
# MUST use global mode because we aren't guaranteed to get PendingTransactionFetcher's pid back fast enough to `allow` # MUST use global mode because we aren't guaranteed to get PendingTransactionFetcher's pid back fast enough to `allow`
# it to use expectations and stubs from test's pid. # it to use expectations and stubs from test's pid.
@ -17,7 +16,7 @@ defmodule Indexer.InternalTransaction.FetcherTest do
@moduletag [capture_log: true, no_geth: true] @moduletag [capture_log: true, no_geth: true]
test "does not try to fetch pending transactions from Indexer.PendingTransaction.Fetcher", %{ test "does not try to fetch pending transactions from Indexer.Fetcher.PendingTransaction", %{
json_rpc_named_arguments: json_rpc_named_arguments json_rpc_named_arguments: json_rpc_named_arguments
} do } do
if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do
@ -71,9 +70,9 @@ defmodule Indexer.InternalTransaction.FetcherTest do
end) end)
hash_strings = hash_strings =
InternalTransaction.Fetcher.init([], fn hash_string, acc -> [hash_string | acc] end, json_rpc_named_arguments) InternalTransaction.init([], fn hash_string, acc -> [hash_string | acc] end, json_rpc_named_arguments)
assert :ok = InternalTransaction.Fetcher.run(hash_strings, json_rpc_named_arguments) assert :ok = InternalTransaction.run(hash_strings, json_rpc_named_arguments)
end end
@tag :no_geth @tag :no_geth
@ -102,9 +101,9 @@ defmodule Indexer.InternalTransaction.FetcherTest do
block_number = 1_000_006 block_number = 1_000_006
insert(:block, number: block_number) insert(:block, number: block_number)
assert :ok = InternalTransaction.Fetcher.run([block_number], json_rpc_named_arguments) assert :ok = InternalTransaction.run([block_number], json_rpc_named_arguments)
assert InternalTransaction.Fetcher.init( assert InternalTransaction.init(
[], [],
fn block_number, acc -> [block_number | acc] end, fn block_number, acc -> [block_number | acc] end,
json_rpc_named_arguments json_rpc_named_arguments
@ -115,7 +114,7 @@ defmodule Indexer.InternalTransaction.FetcherTest do
test "does not buffer pending transactions", %{json_rpc_named_arguments: json_rpc_named_arguments} do test "does not buffer pending transactions", %{json_rpc_named_arguments: json_rpc_named_arguments} do
insert(:transaction) insert(:transaction)
assert InternalTransaction.Fetcher.init( assert InternalTransaction.init(
[], [],
fn hash_string, acc -> [hash_string | acc] end, fn hash_string, acc -> [hash_string | acc] end,
json_rpc_named_arguments json_rpc_named_arguments
@ -133,7 +132,7 @@ defmodule Indexer.InternalTransaction.FetcherTest do
|> insert() |> insert()
|> with_block(block) |> with_block(block)
assert InternalTransaction.Fetcher.init( assert InternalTransaction.init(
[], [],
fn hash_string, acc -> [hash_string | acc] end, fn hash_string, acc -> [hash_string | acc] end,
json_rpc_named_arguments json_rpc_named_arguments
@ -148,7 +147,7 @@ defmodule Indexer.InternalTransaction.FetcherTest do
|> insert() |> insert()
|> with_block(internal_transactions_indexed_at: DateTime.utc_now()) |> with_block(internal_transactions_indexed_at: DateTime.utc_now())
assert InternalTransaction.Fetcher.init( assert InternalTransaction.init(
[], [],
fn hash_string, acc -> [hash_string | acc] end, fn hash_string, acc -> [hash_string | acc] end,
json_rpc_named_arguments json_rpc_named_arguments
@ -161,7 +160,7 @@ defmodule Indexer.InternalTransaction.FetcherTest do
} do } do
block = insert(:block) block = insert(:block)
assert InternalTransaction.Fetcher.init( assert InternalTransaction.init(
[], [],
fn block_number, acc -> [block_number | acc] end, fn block_number, acc -> [block_number | acc] end,
json_rpc_named_arguments json_rpc_named_arguments
@ -174,7 +173,7 @@ defmodule Indexer.InternalTransaction.FetcherTest do
} do } do
insert(:block, internal_transactions_indexed_at: DateTime.utc_now()) insert(:block, internal_transactions_indexed_at: DateTime.utc_now())
assert InternalTransaction.Fetcher.init( assert InternalTransaction.init(
[], [],
fn block_number, acc -> [block_number | acc] end, fn block_number, acc -> [block_number | acc] end,
json_rpc_named_arguments json_rpc_named_arguments
@ -198,7 +197,7 @@ defmodule Indexer.InternalTransaction.FetcherTest do
log = log =
capture_log(fn -> capture_log(fn ->
InternalTransaction.Fetcher.run( InternalTransaction.run(
[ [
{1, bytes, 0}, {1, bytes, 0},
{1, bytes, 0} {1, bytes, 0}
@ -276,7 +275,7 @@ defmodule Indexer.InternalTransaction.FetcherTest do
|> with_block() |> with_block()
:ok = :ok =
InternalTransaction.Fetcher.run( InternalTransaction.run(
[ [
{7_202_692, bytes, 0} {7_202_692, bytes, 0}
], ],
@ -305,7 +304,7 @@ defmodule Indexer.InternalTransaction.FetcherTest do
%Transaction{hash: %Hash{bytes: bytes}} = %Transaction{hash: %Hash{bytes: bytes}} =
insert(:transaction, hash: "0x0000000000000000000000000000000000000000000000000000000000000001") insert(:transaction, hash: "0x0000000000000000000000000000000000000000000000000000000000000001")
assert InternalTransaction.Fetcher.run( assert InternalTransaction.run(
[ [
{1, bytes, 0}, {1, bytes, 0},
{1, bytes, 0} {1, bytes, 0}

@ -1,4 +1,4 @@
defmodule Indexer.PendingTransactionFetcherTest do defmodule Indexer.Fetcher.PendingTransactionTest do
# `async: false` due to use of named GenServer # `async: false` due to use of named GenServer
use EthereumJSONRPC.Case, async: false use EthereumJSONRPC.Case, async: false
use Explorer.DataCase use Explorer.DataCase
@ -6,7 +6,7 @@ defmodule Indexer.PendingTransactionFetcherTest do
import Mox import Mox
alias Explorer.Chain.Transaction alias Explorer.Chain.Transaction
alias Indexer.PendingTransaction alias Indexer.Fetcher.PendingTransaction
# MUST use global mode because we aren't guaranteed to get PendingTransactionFetcher's pid back fast enough to `allow` # MUST use global mode because we aren't guaranteed to get PendingTransactionFetcher's pid back fast enough to `allow`
# it to use expectations and stubs from test's pid. # it to use expectations and stubs from test's pid.

@ -1,8 +1,8 @@
defmodule Indexer.ReplacedTransaction.FetcherTest do defmodule Indexer.Fetcher.ReplacedTransactionTest do
use Explorer.DataCase use Explorer.DataCase
alias Explorer.Chain.{Transaction} alias Explorer.Chain.{Transaction}
alias Indexer.ReplacedTransaction alias Indexer.Fetcher.ReplacedTransaction
@moduletag :capture_log @moduletag :capture_log
@ -71,7 +71,7 @@ defmodule Indexer.ReplacedTransaction.FetcherTest do
ReplacedTransaction.Supervisor.Case.start_supervised!() ReplacedTransaction.Supervisor.Case.start_supervised!()
assert :ok = assert :ok =
ReplacedTransaction.Fetcher.async_fetch([ ReplacedTransaction.async_fetch([
%{ %{
block_hash: mined_transaction.block_hash, block_hash: mined_transaction.block_hash,
nonce: mined_transaction.nonce, nonce: mined_transaction.nonce,
@ -160,7 +160,7 @@ defmodule Indexer.ReplacedTransaction.FetcherTest do
ReplacedTransaction.Supervisor.Case.start_supervised!() ReplacedTransaction.Supervisor.Case.start_supervised!()
# assert :ok = # assert :ok =
# ReplacedTransaction.Fetcher.async_fetch([ # ReplacedTransaction.async_fetch([
# %{ # %{
# block_hash: mined_transaction.block_hash, # block_hash: mined_transaction.block_hash,
# nonce: mined_transaction.nonce, # nonce: mined_transaction.nonce,

@ -1,11 +1,11 @@
defmodule Indexer.TokenBalance.FetcherTest do defmodule Indexer.Fetcher.TokenBalanceTest do
use EthereumJSONRPC.Case use EthereumJSONRPC.Case
use Explorer.DataCase use Explorer.DataCase
import Mox import Mox
alias Explorer.Chain.{Address, Hash} alias Explorer.Chain.{Address, Hash}
alias Indexer.TokenBalance alias Indexer.Fetcher.TokenBalance
@moduletag :capture_log @moduletag :capture_log
@ -22,7 +22,7 @@ defmodule Indexer.TokenBalance.FetcherTest do
insert(:token_balance, value_fetched_at: DateTime.utc_now()) insert(:token_balance, value_fetched_at: DateTime.utc_now())
assert TokenBalance.Fetcher.init([], &[&1 | &2], nil) == [ assert TokenBalance.init([], &[&1 | &2], nil) == [
{address_hash_bytes, token_contract_address_hash_bytes, block_number, 0} {address_hash_bytes, token_contract_address_hash_bytes, block_number, 0}
] ]
end end
@ -57,7 +57,7 @@ defmodule Indexer.TokenBalance.FetcherTest do
end end
) )
assert TokenBalance.Fetcher.run( assert TokenBalance.run(
[{address_hash_bytes, token_contract_address_hash_bytes, block_number, 0}], [{address_hash_bytes, token_contract_address_hash_bytes, block_number, 0}],
nil nil
) == :ok ) == :ok
@ -109,13 +109,13 @@ defmodule Indexer.TokenBalance.FetcherTest do
} }
] ]
assert TokenBalance.Fetcher.run(token_balances, nil) == :ok assert TokenBalance.run(token_balances, nil) == :ok
end end
end end
describe "import_token_balances/1" do describe "import_token_balances/1" do
test "ignores when it receives a empty list" do test "ignores when it receives a empty list" do
assert TokenBalance.Fetcher.import_token_balances([]) == :ok assert TokenBalance.import_token_balances([]) == :ok
end end
test "returns :error when the token balances has invalid data" do test "returns :error when the token balances has invalid data" do
@ -131,7 +131,7 @@ defmodule Indexer.TokenBalance.FetcherTest do
} }
] ]
assert TokenBalance.Fetcher.import_token_balances(token_balances_params) == :error assert TokenBalance.import_token_balances(token_balances_params) == :error
end end
test "insert the missing address, import the token balances and return :ok when the address does not exist yet" do test "insert the missing address, import the token balances and return :ok when the address does not exist yet" do
@ -147,7 +147,7 @@ defmodule Indexer.TokenBalance.FetcherTest do
] ]
{:ok, address_hash} = Explorer.Chain.string_to_address_hash("0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF") {:ok, address_hash} = Explorer.Chain.string_to_address_hash("0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF")
assert TokenBalance.Fetcher.import_token_balances(token_balances_params) == :ok assert TokenBalance.import_token_balances(token_balances_params) == :ok
assert {:ok, _} = Explorer.Chain.hash_to_address(address_hash) assert {:ok, _} = Explorer.Chain.hash_to_address(address_hash)
end end
@ -169,7 +169,7 @@ defmodule Indexer.TokenBalance.FetcherTest do
} }
] ]
assert TokenBalance.Fetcher.import_token_balances(token_balances_params) == :ok assert TokenBalance.import_token_balances(token_balances_params) == :ok
end end
end end
end end

@ -1,4 +1,4 @@
defmodule Indexer.Token.FetcherTest do defmodule Indexer.Fetcher.TokenTest do
use EthereumJSONRPC.Case use EthereumJSONRPC.Case
use Explorer.DataCase use Explorer.DataCase
@ -6,7 +6,7 @@ defmodule Indexer.Token.FetcherTest do
alias Explorer.Chain alias Explorer.Chain
alias Explorer.Chain.Token alias Explorer.Chain.Token
alias Indexer.Token.Fetcher alias Indexer.Fetcher.Token, as: TokenFetcher
setup :verify_on_exit! setup :verify_on_exit!
@ -15,7 +15,7 @@ defmodule Indexer.Token.FetcherTest do
insert(:token, cataloged: true) insert(:token, cataloged: true)
%Token{contract_address_hash: uncatalog_address} = insert(:token, cataloged: false) %Token{contract_address_hash: uncatalog_address} = insert(:token, cataloged: false)
assert Fetcher.init([], &[&1 | &2], json_rpc_named_arguments) == [uncatalog_address] assert TokenFetcher.init([], &[&1 | &2], json_rpc_named_arguments) == [uncatalog_address]
end end
end end
@ -61,7 +61,7 @@ defmodule Indexer.Token.FetcherTest do
end end
) )
assert Fetcher.run([contract_address_hash], json_rpc_named_arguments) == :ok assert TokenFetcher.run([contract_address_hash], json_rpc_named_arguments) == :ok
expected_supply = Decimal.new(1_000_000_000_000_000_000) expected_supply = Decimal.new(1_000_000_000_000_000_000)

@ -1,11 +1,11 @@
defmodule Indexer.Token.MetadataUpdaterTest do defmodule Indexer.Fetcher.TokenUpdaterTest do
use Explorer.DataCase use Explorer.DataCase
import Mox import Mox
alias Explorer.Chain alias Explorer.Chain
alias Explorer.Chain.Token alias Explorer.Chain.Token
alias Indexer.Token.MetadataUpdater alias Indexer.Fetcher.TokenUpdater
setup :verify_on_exit! setup :verify_on_exit!
setup :set_mox_global setup :set_mox_global
@ -49,7 +49,7 @@ defmodule Indexer.Token.MetadataUpdaterTest do
end end
) )
pid = start_supervised!({MetadataUpdater, %{update_interval: 1}}) pid = start_supervised!({TokenUpdater, [%{update_interval: 1}, []]})
wait_for_results(fn -> wait_for_results(fn ->
updated = Repo.one!(from(t in Token, where: t.cataloged == true and not is_nil(t.name), limit: 1)) updated = Repo.one!(from(t in Token, where: t.cataloged == true and not is_nil(t.name), limit: 1))
@ -102,7 +102,7 @@ defmodule Indexer.Token.MetadataUpdaterTest do
end end
) )
MetadataUpdater.update_metadata([token.contract_address_hash]) TokenUpdater.update_metadata([token.contract_address_hash])
expected_supply = Decimal.new(1_000_000_000_000_000_000) expected_supply = Decimal.new(1_000_000_000_000_000_000)

@ -1,4 +1,4 @@
defmodule Indexer.Block.Uncle.FetcherTest do defmodule Indexer.Fetcher.UncleBlockTest do
# MUST be `async: false` so that {:shared, pid} is set for connection to allow CoinBalanceFetcher's self-send to have # MUST be `async: false` so that {:shared, pid} is set for connection to allow CoinBalanceFetcher's self-send to have
# connection allowed immediately. # connection allowed immediately.
use EthereumJSONRPC.Case, async: false use EthereumJSONRPC.Case, async: false
@ -6,6 +6,7 @@ defmodule Indexer.Block.Uncle.FetcherTest do
alias Explorer.Chain alias Explorer.Chain
alias Indexer.Block alias Indexer.Block
alias Indexer.Fetcher.UncleBlock
import Mox import Mox
@ -32,10 +33,10 @@ defmodule Indexer.Block.Uncle.FetcherTest do
describe "child_spec/1" do describe "child_spec/1" do
test "raises ArgumentError is `json_rpc_named_arguments is not provided" do test "raises ArgumentError is `json_rpc_named_arguments is not provided" do
assert_raise ArgumentError, assert_raise ArgumentError,
":json_rpc_named_arguments must be provided to `Elixir.Indexer.Block.Uncle.Fetcher.child_spec " <> ":json_rpc_named_arguments must be provided to `Elixir.Indexer.Fetcher.UncleBlock.child_spec " <>
"to allow for json_rpc calls when running.", "to allow for json_rpc calls when running.",
fn -> fn ->
start_supervised({Block.Uncle.Fetcher, [[], []]}) start_supervised({UncleBlock, [[], []]})
end end
end end
end end
@ -104,7 +105,7 @@ defmodule Indexer.Block.Uncle.FetcherTest do
]} ]}
end) end)
Block.Uncle.Supervisor.Case.start_supervised!( UncleBlock.Supervisor.Case.start_supervised!(
block_fetcher: %Block.Fetcher{json_rpc_named_arguments: json_rpc_named_arguments} block_fetcher: %Block.Fetcher{json_rpc_named_arguments: json_rpc_named_arguments}
) )

@ -8,11 +8,10 @@ defmodule Indexer.Temporary.AddressesWithoutCodeTest do
alias Explorer.Repo alias Explorer.Repo
alias Explorer.Chain.{Address, Transaction} alias Explorer.Chain.{Address, Transaction}
alias Indexer.Temporary.AddressesWithoutCode.Supervisor
alias Indexer.CoinBalance
alias Indexer.Block.Fetcher alias Indexer.Block.Fetcher
alias Indexer.Block.Realtime.Fetcher, as: RealtimeFetcher alias Indexer.Block.Realtime.Fetcher, as: RealtimeFetcher
alias Indexer.{CoinBalance, Code, InternalTransaction, ReplacedTransaction, Token, TokenBalance} alias Indexer.Fetcher.{CoinBalance, ContractCode, InternalTransaction, ReplacedTransaction, Token, TokenBalance}
alias Indexer.Temporary.AddressesWithoutCode.Supervisor
@moduletag capture_log: true @moduletag capture_log: true
@ -23,7 +22,7 @@ defmodule Indexer.Temporary.AddressesWithoutCodeTest do
describe "run/1" do describe "run/1" do
setup %{json_rpc_named_arguments: json_rpc_named_arguments} do setup %{json_rpc_named_arguments: json_rpc_named_arguments} do
CoinBalance.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) CoinBalance.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments)
Code.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) ContractCode.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments)
InternalTransaction.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) InternalTransaction.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments)
Token.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) Token.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments)
TokenBalance.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) TokenBalance.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments)

@ -8,8 +8,8 @@ defmodule Indexer.Temporary.FailedCreatedAddressesTest do
alias Explorer.Repo alias Explorer.Repo
alias Explorer.Chain.Address alias Explorer.Chain.Address
alias Indexer.Fetcher.CoinBalance
alias Indexer.Temporary.FailedCreatedAddresses.Supervisor alias Indexer.Temporary.FailedCreatedAddresses.Supervisor
alias Indexer.CoinBalance
@moduletag capture_log: true @moduletag capture_log: true

@ -1,21 +1,24 @@
defmodule Indexer.TokenTransfer.Uncataloged.WorkerTest do defmodule Indexer.Temporary.UncatalogedTokenTransfersTest do
use Explorer.DataCase use Explorer.DataCase
alias Indexer.Sequence alias Indexer.Block.Catchup.Sequence
alias Indexer.TokenTransfer.Uncataloged.{Worker, TaskSupervisor} alias Indexer.Temporary.UncatalogedTokenTransfers
@moduletag :capture_log @moduletag :capture_log
describe "start_link/1" do describe "start_link/1" do
test "starts the worker" do test "starts the worker" do
assert {:ok, _pid} = Worker.start_link(supervisor: self()) assert {:ok, _pid} = UncatalogedTokenTransfers.start_link(supervisor: self())
end end
end end
describe "init/1" do describe "init/1" do
test "sends message to self" do test "sends message to self" do
pid = self() pid = self()
assert {:ok, %{task_ref: nil, block_numbers: [], sup_pid: ^pid}} = Worker.init(supervisor: self())
assert {:ok, %{task_ref: nil, block_numbers: [], sup_pid: ^pid}} =
UncatalogedTokenTransfers.init(supervisor: self())
assert_received :scan assert_received :scan
end end
end end
@ -23,7 +26,7 @@ defmodule Indexer.TokenTransfer.Uncataloged.WorkerTest do
describe "handle_info with :scan" do describe "handle_info with :scan" do
test "sends shutdown to supervisor" do test "sends shutdown to supervisor" do
state = %{task_ref: nil, block_numbers: [], sup_pid: self()} state = %{task_ref: nil, block_numbers: [], sup_pid: self()}
Task.async(fn -> Worker.handle_info(:scan, state) end) Task.async(fn -> UncatalogedTokenTransfers.handle_info(:scan, state) end)
assert_receive {_, _, {:terminate, :normal}}, 200 assert_receive {_, _, {:terminate, :normal}}, 200
end end
@ -49,18 +52,18 @@ defmodule Indexer.TokenTransfer.Uncataloged.WorkerTest do
expected_state = %{task_ref: nil, block_numbers: [block_number], retry_interval: 1} expected_state = %{task_ref: nil, block_numbers: [block_number], retry_interval: 1}
state = %{task_ref: nil, block_numbers: [], retry_interval: 1} state = %{task_ref: nil, block_numbers: [], retry_interval: 1}
assert {:noreply, ^expected_state} = Worker.handle_info(:scan, state) assert {:noreply, ^expected_state} = UncatalogedTokenTransfers.handle_info(:scan, state)
assert_receive :push_front_blocks assert_receive :push_front_blocks
end end
end end
describe "handle_info with :push_front_blocks" do describe "handle_info with :push_front_blocks" do
test "starts a task" do test "starts a task" do
task_sup_pid = start_supervised!({Task.Supervisor, name: TaskSupervisor}) task_sup_pid = start_supervised!({Task.Supervisor, name: UncatalogedTokenTransfers.TaskSupervisor})
start_supervised!({Sequence, [[ranges: [], step: -1], [name: :block_catchup_sequencer]]}) start_supervised!({Sequence, [[ranges: [], step: -1], [name: :block_catchup_sequencer]]})
state = %{task_ref: nil, block_numbers: [1]} state = %{task_ref: nil, block_numbers: [1]}
assert {:noreply, %{task_ref: task_ref}} = Worker.handle_info(:push_front_blocks, state) assert {:noreply, %{task_ref: task_ref}} = UncatalogedTokenTransfers.handle_info(:push_front_blocks, state)
assert is_reference(task_ref) assert is_reference(task_ref)
refute_receive {^task_ref, {:error, :queue_unavailable}} refute_receive {^task_ref, {:error, :queue_unavailable}}
@ -74,7 +77,7 @@ defmodule Indexer.TokenTransfer.Uncataloged.WorkerTest do
test "sends shutdown to supervisor on success" do test "sends shutdown to supervisor on success" do
ref = Process.monitor(self()) ref = Process.monitor(self())
state = %{task_ref: ref, block_numbers: [], sup_pid: self()} state = %{task_ref: ref, block_numbers: [], sup_pid: self()}
Task.async(fn -> assert Worker.handle_info({ref, :ok}, state) end) Task.async(fn -> assert UncatalogedTokenTransfers.handle_info({ref, :ok}, state) end)
assert_receive {_, _, {:terminate, :normal}} assert_receive {_, _, {:terminate, :normal}}
end end
@ -82,7 +85,10 @@ defmodule Indexer.TokenTransfer.Uncataloged.WorkerTest do
ref = Process.monitor(self()) ref = Process.monitor(self())
state = %{task_ref: ref, block_numbers: [1], sup_pid: self(), retry_interval: 1} state = %{task_ref: ref, block_numbers: [1], sup_pid: self(), retry_interval: 1}
expected_state = %{state | task_ref: nil} expected_state = %{state | task_ref: nil}
assert {:noreply, ^expected_state} = Worker.handle_info({ref, {:error, :queue_unavailable}}, state)
assert {:noreply, ^expected_state} =
UncatalogedTokenTransfers.handle_info({ref, {:error, :queue_unavailable}}, state)
assert_receive :push_front_blocks assert_receive :push_front_blocks
end end
end end
@ -91,7 +97,10 @@ defmodule Indexer.TokenTransfer.Uncataloged.WorkerTest do
test "sends message to self to try again" do test "sends message to self to try again" do
ref = Process.monitor(self()) ref = Process.monitor(self())
state = %{task_ref: ref, block_numbers: [1], sup_pid: self(), retry_interval: 1} state = %{task_ref: ref, block_numbers: [1], sup_pid: self(), retry_interval: 1}
assert {:noreply, %{task_ref: nil}} = Worker.handle_info({:DOWN, ref, :process, self(), :EXIT}, state)
assert {:noreply, %{task_ref: nil}} =
UncatalogedTokenTransfers.handle_info({:DOWN, ref, :process, self(), :EXIT}, state)
assert_receive :push_front_blocks assert_receive :push_front_blocks
end end
end end

@ -5,7 +5,7 @@ defmodule Indexer.TokenBalancesTest do
doctest Indexer.TokenBalances doctest Indexer.TokenBalances
alias Indexer.TokenBalances alias Indexer.TokenBalances
alias Indexer.TokenBalance alias Indexer.Fetcher.TokenBalance
alias Explorer.Chain.Hash alias Explorer.Chain.Hash
import Mox import Mox

@ -1,8 +1,8 @@
defmodule Indexer.Address.CoinBalancesTest do defmodule Indexer.Transform.AddressCoinBalancesTest do
use ExUnit.Case, async: true use ExUnit.Case, async: true
alias Explorer.Factory alias Explorer.Factory
alias Indexer.Address.CoinBalances alias Indexer.Transform.AddressCoinBalances
describe "params_set/1" do describe "params_set/1" do
test "with block extracts miner_hash" do test "with block extracts miner_hash" do
@ -12,7 +12,7 @@ defmodule Indexer.Address.CoinBalancesTest do
block_number = 1 block_number = 1
params_set = CoinBalances.params_set(%{blocks_params: [%{miner_hash: miner_hash, number: block_number}]}) params_set = AddressCoinBalances.params_set(%{blocks_params: [%{miner_hash: miner_hash, number: block_number}]})
assert MapSet.size(params_set) == 1 assert MapSet.size(params_set) == 1
assert %{address_hash: miner_hash, block_number: block_number} assert %{address_hash: miner_hash, block_number: block_number}
@ -20,7 +20,7 @@ defmodule Indexer.Address.CoinBalancesTest do
test "with block second degree relations extracts nothing" do test "with block second degree relations extracts nothing" do
params_set = params_set =
CoinBalances.params_set(%{ AddressCoinBalances.params_set(%{
block_second_degree_relations_params: [%{nephew_hash: Factory.block_hash(), uncle_hash: Factory.block_hash()}] block_second_degree_relations_params: [%{nephew_hash: Factory.block_hash(), uncle_hash: Factory.block_hash()}]
}) })
@ -34,7 +34,7 @@ defmodule Indexer.Address.CoinBalancesTest do
|> Map.update!(:type, &to_string/1) |> Map.update!(:type, &to_string/1)
|> Map.put(:block_number, 1) |> Map.put(:block_number, 1)
params_set = CoinBalances.params_set(%{internal_transactions_params: [internal_transaction_params]}) params_set = AddressCoinBalances.params_set(%{internal_transactions_params: [internal_transaction_params]})
assert MapSet.size(params_set) == 0 assert MapSet.size(params_set) == 0
end end
@ -47,7 +47,7 @@ defmodule Indexer.Address.CoinBalancesTest do
|> Map.put(:block_number, 1) |> Map.put(:block_number, 1)
|> Map.put(:error, "illegal operation") |> Map.put(:error, "illegal operation")
params_set = CoinBalances.params_set(%{internal_transactions_params: [internal_transaction_params]}) params_set = AddressCoinBalances.params_set(%{internal_transactions_params: [internal_transaction_params]})
assert MapSet.size(params_set) == 0 assert MapSet.size(params_set) == 0
end end
@ -66,7 +66,7 @@ defmodule Indexer.Address.CoinBalancesTest do
|> Map.put(:block_number, block_number) |> Map.put(:block_number, block_number)
|> Map.put(:created_contract_address_hash, created_contract_address_hash) |> Map.put(:created_contract_address_hash, created_contract_address_hash)
params_set = CoinBalances.params_set(%{internal_transactions_params: [internal_transaction_params]}) params_set = AddressCoinBalances.params_set(%{internal_transactions_params: [internal_transaction_params]})
assert MapSet.size(params_set) == 1 assert MapSet.size(params_set) == 1
assert %{address_hash: created_contract_address_hash, block_number: block_number} assert %{address_hash: created_contract_address_hash, block_number: block_number}
@ -91,7 +91,7 @@ defmodule Indexer.Address.CoinBalancesTest do
|> Map.put(:from_address_hash, from_address_hash) |> Map.put(:from_address_hash, from_address_hash)
|> Map.put(:to_address_hash, to_address_hash) |> Map.put(:to_address_hash, to_address_hash)
params_set = CoinBalances.params_set(%{internal_transactions_params: [internal_transaction_params]}) params_set = AddressCoinBalances.params_set(%{internal_transactions_params: [internal_transaction_params]})
assert MapSet.size(params_set) == 2 assert MapSet.size(params_set) == 2
assert %{address_hash: from_address_hash, block_number: block_number} assert %{address_hash: from_address_hash, block_number: block_number}
@ -111,7 +111,7 @@ defmodule Indexer.Address.CoinBalancesTest do
|> Map.put(:block_number, block_number) |> Map.put(:block_number, block_number)
|> Map.put(:address_hash, address_hash) |> Map.put(:address_hash, address_hash)
params_set = CoinBalances.params_set(%{logs_params: [log_params]}) params_set = AddressCoinBalances.params_set(%{logs_params: [log_params]})
assert MapSet.size(params_set) == 1 assert MapSet.size(params_set) == 1
assert MapSet.new([%{address_hash: address_hash, block_number: block_number}]) == params_set assert MapSet.new([%{address_hash: address_hash, block_number: block_number}]) == params_set
@ -137,7 +137,7 @@ defmodule Indexer.Address.CoinBalancesTest do
|> Map.put(:block_number, block_number) |> Map.put(:block_number, block_number)
|> Map.put(:address_hash, address_hash) |> Map.put(:address_hash, address_hash)
params_set = CoinBalances.params_set(%{logs_params: [log_params1, log_params2]}) params_set = AddressCoinBalances.params_set(%{logs_params: [log_params1, log_params2]})
assert MapSet.size(params_set) == 1 assert MapSet.size(params_set) == 1
assert MapSet.new([%{address_hash: address_hash, block_number: block_number}]) == params_set assert MapSet.new([%{address_hash: address_hash, block_number: block_number}]) == params_set
@ -156,7 +156,7 @@ defmodule Indexer.Address.CoinBalancesTest do
|> Map.put(:block_number, block_number) |> Map.put(:block_number, block_number)
|> Map.put(:from_address_hash, from_address_hash) |> Map.put(:from_address_hash, from_address_hash)
params_set = CoinBalances.params_set(%{transactions_params: [transaction_params]}) params_set = AddressCoinBalances.params_set(%{transactions_params: [transaction_params]})
assert MapSet.size(params_set) == 1 assert MapSet.size(params_set) == 1
assert %{address_hash: from_address_hash, block_number: block_number} assert %{address_hash: from_address_hash, block_number: block_number}
@ -180,7 +180,7 @@ defmodule Indexer.Address.CoinBalancesTest do
|> Map.put(:from_address_hash, from_address_hash) |> Map.put(:from_address_hash, from_address_hash)
|> Map.put(:to_address_hash, to_address_hash) |> Map.put(:to_address_hash, to_address_hash)
params_set = CoinBalances.params_set(%{transactions_params: [transaction_params]}) params_set = AddressCoinBalances.params_set(%{transactions_params: [transaction_params]})
assert MapSet.size(params_set) == 2 assert MapSet.size(params_set) == 2
assert %{address_hash: from_address_hash, block_number: block_number} assert %{address_hash: from_address_hash, block_number: block_number}

@ -1,8 +1,8 @@
defmodule Indexer.Address.TokenBalancesTest do defmodule Indexer.Transform.AddressTokenBalancesTest do
use ExUnit.Case, async: true use ExUnit.Case, async: true
alias Explorer.Factory alias Explorer.Factory
alias Indexer.Address.TokenBalances alias Indexer.Transform.AddressTokenBalances
describe "params_set/1" do describe "params_set/1" do
test "with token transfer extract from_address, to_address, and token_contract_address_hash" do test "with token transfer extract from_address, to_address, and token_contract_address_hash" do
@ -27,7 +27,7 @@ defmodule Indexer.Address.TokenBalancesTest do
token_contract_address_hash: token_contract_address_hash token_contract_address_hash: token_contract_address_hash
} }
params_set = TokenBalances.params_set(%{token_transfers_params: [token_transfer_params]}) params_set = AddressTokenBalances.params_set(%{token_transfers_params: [token_transfer_params]})
assert MapSet.size(params_set) == 2 assert MapSet.size(params_set) == 2
assert %{address_hash: from_address_hash, block_number: block_number} assert %{address_hash: from_address_hash, block_number: block_number}
@ -49,7 +49,7 @@ defmodule Indexer.Address.TokenBalancesTest do
token_type: "ERC-721" token_type: "ERC-721"
} }
params_set = TokenBalances.params_set(%{token_transfers_params: [token_transfer_params]}) params_set = AddressTokenBalances.params_set(%{token_transfers_params: [token_transfer_params]})
assert params_set == assert params_set ==
MapSet.new([ MapSet.new([
@ -75,7 +75,7 @@ defmodule Indexer.Address.TokenBalancesTest do
token_type: "ERC-721" token_type: "ERC-721"
} }
params_set = TokenBalances.params_set(%{token_transfers_params: [token_transfer_params]}) params_set = AddressTokenBalances.params_set(%{token_transfers_params: [token_transfer_params]})
assert MapSet.size(params_set) == 0 assert MapSet.size(params_set) == 0
end end

@ -1,13 +1,13 @@
defmodule Indexer.AddressExtractionTest do defmodule Indexer.Transform.AddressesTest do
use Explorer.DataCase, async: true use Explorer.DataCase, async: true
alias Indexer.AddressExtraction alias Indexer.Transform.Addresses
doctest AddressExtraction doctest Addresses
describe "extract_addresses/1" do describe "extract_addresses/1" do
test "blocks without a `miner_hash` aren't extracted" do test "blocks without a `miner_hash` aren't extracted" do
assert Indexer.AddressExtraction.extract_addresses(%{ assert Addresses.extract_addresses(%{
blocks: [ blocks: [
%{ %{
number: 34 number: 34
@ -17,7 +17,7 @@ defmodule Indexer.AddressExtractionTest do
end end
test "blocks without a `number` aren't extracted" do test "blocks without a `number` aren't extracted" do
assert Indexer.AddressExtraction.extract_addresses(%{ assert Addresses.extract_addresses(%{
blocks: [ blocks: [
%{ %{
miner_hash: "0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca" miner_hash: "0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca"
@ -27,7 +27,7 @@ defmodule Indexer.AddressExtractionTest do
end end
test "internal_transactions with a `from_address_hash` without a `block_number` aren't extracted" do test "internal_transactions with a `from_address_hash` without a `block_number` aren't extracted" do
assert Indexer.AddressExtraction.extract_addresses(%{ assert Addresses.extract_addresses(%{
internal_transactions: [ internal_transactions: [
%{ %{
from_address_hash: "0x0000000000000000000000000000000000000001" from_address_hash: "0x0000000000000000000000000000000000000001"
@ -37,7 +37,7 @@ defmodule Indexer.AddressExtractionTest do
end end
test "internal_transactions with a `to_address_hash` without a `block_number` aren't extracted" do test "internal_transactions with a `to_address_hash` without a `block_number` aren't extracted" do
assert Indexer.AddressExtraction.extract_addresses(%{ assert Addresses.extract_addresses(%{
internal_transactions: [ internal_transactions: [
%{ %{
to_address_hash: "0x0000000000000000000000000000000000000002" to_address_hash: "0x0000000000000000000000000000000000000002"
@ -48,7 +48,7 @@ defmodule Indexer.AddressExtractionTest do
test "internal_transactions with a `created_contract_address_hash` and `created_contract_code` " <> test "internal_transactions with a `created_contract_address_hash` and `created_contract_code` " <>
"without a `block_number` aren't extracted" do "without a `block_number` aren't extracted" do
assert Indexer.AddressExtraction.extract_addresses(%{ assert Addresses.extract_addresses(%{
internal_transactions: [ internal_transactions: [
%{ %{
created_contract_address_hash: "0x0000000000000000000000000000000000000003", created_contract_address_hash: "0x0000000000000000000000000000000000000003",
@ -59,7 +59,7 @@ defmodule Indexer.AddressExtractionTest do
end end
test "differing contract code is ignored" do test "differing contract code is ignored" do
assert Indexer.AddressExtraction.extract_addresses(%{ assert Addresses.extract_addresses(%{
internal_transactions: [ internal_transactions: [
%{ %{
block_number: 1, block_number: 1,
@ -98,7 +98,7 @@ defmodule Indexer.AddressExtractionTest do
] ]
} }
assert Indexer.AddressExtraction.extract_addresses(code_params) == [ assert Addresses.extract_addresses(code_params) == [
%{ %{
contract_code: contract_code:
"0x600160008035811a818181146012578301005b601b6001356025565b8060005260206000f25b600060078202905091905056", "0x600160008035811a818181146012578301005b601b6001356025565b8060005260206000f25b600060078202905091905056",
@ -153,7 +153,7 @@ defmodule Indexer.AddressExtractionTest do
block_reward_contract_beneficiaries: [beneficiary] block_reward_contract_beneficiaries: [beneficiary]
} }
assert AddressExtraction.extract_addresses(blockchain_data) == [ assert Addresses.extract_addresses(blockchain_data) == [
%{hash: block.miner_hash, fetched_coin_balance_block_number: block.number}, %{hash: block.miner_hash, fetched_coin_balance_block_number: block.number},
%{ %{
hash: internal_transaction.from_address_hash, hash: internal_transaction.from_address_hash,
@ -202,7 +202,7 @@ defmodule Indexer.AddressExtractionTest do
logs: [] logs: []
} }
addresses = AddressExtraction.extract_addresses(empty_blockchain_data) addresses = Addresses.extract_addresses(empty_blockchain_data)
assert Enum.empty?(addresses) assert Enum.empty?(addresses)
end end
@ -222,7 +222,7 @@ defmodule Indexer.AddressExtractionTest do
] ]
} }
assert AddressExtraction.extract_addresses(blockchain_data) == assert Addresses.extract_addresses(blockchain_data) ==
[ [
%{hash: hash, fetched_coin_balance_block_number: 34, contract_code: "code", nonce: 12} %{hash: hash, fetched_coin_balance_block_number: 34, contract_code: "code", nonce: 12}
] ]
@ -234,13 +234,13 @@ defmodule Indexer.AddressExtractionTest do
unkown_entity: [%{hash: "0x8bf38d4764929064f2d4d3a56520a76ab3df415b"}] unkown_entity: [%{hash: "0x8bf38d4764929064f2d4d3a56520a76ab3df415b"}]
} }
assert AddressExtraction.extract_addresses(blockchain_data) == [ assert Addresses.extract_addresses(blockchain_data) == [
%{fetched_coin_balance_block_number: 34, hash: "0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca"} %{fetched_coin_balance_block_number: 34, hash: "0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca"}
] ]
end end
test "returns an empty list when there isn't a recognized entity" do test "returns an empty list when there isn't a recognized entity" do
addresses = AddressExtraction.extract_addresses(%{}) addresses = Addresses.extract_addresses(%{})
assert Enum.empty?(addresses) assert Enum.empty?(addresses)
end end
@ -258,7 +258,7 @@ defmodule Indexer.AddressExtractionTest do
%{field_1: "hash1", field_2: "hash3"} %{field_1: "hash1", field_2: "hash3"}
] ]
assert AddressExtraction.extract_addresses_from_collection(items, fields_map, %AddressExtraction{pending: false}) == assert Addresses.extract_addresses_from_collection(items, fields_map, %Addresses{pending: false}) ==
[ [
%{hash: "hash1"}, %{hash: "hash1"},
%{hash: "hash2"}, %{hash: "hash2"},
@ -276,7 +276,7 @@ defmodule Indexer.AddressExtractionTest do
item = %{field_1: "hash1", field_2: "hash2"} item = %{field_1: "hash1", field_2: "hash2"}
response = AddressExtraction.extract_addresses_from_item(item, fields_map, %AddressExtraction{pending: false}) response = Addresses.extract_addresses_from_item(item, fields_map, %Addresses{pending: false})
assert response == [%{hash: "hash1"}] assert response == [%{hash: "hash1"}]
end end
@ -289,7 +289,7 @@ defmodule Indexer.AddressExtractionTest do
item = %{field_1: "hash1", field_2: "hash2"} item = %{field_1: "hash1", field_2: "hash2"}
response = AddressExtraction.extract_addresses_from_item(item, fields_map, %AddressExtraction{pending: false}) response = Addresses.extract_addresses_from_item(item, fields_map, %Addresses{pending: false})
assert response == [%{hash: "hash1"}, %{hash: "hash2"}] assert response == [%{hash: "hash1"}, %{hash: "hash2"}]
end end
@ -306,10 +306,10 @@ defmodule Indexer.AddressExtractionTest do
data = %{field_1: "hash1", field_2: "hash2", field_2_code: "code"} data = %{field_1: "hash1", field_2: "hash2", field_2_code: "code"}
response = response =
AddressExtraction.extract_addresses_from_item( Addresses.extract_addresses_from_item(
data, data,
fields_map, fields_map,
%AddressExtraction{pending: false} %Addresses{pending: false}
) )
assert response == [ assert response == [

@ -1,7 +1,7 @@
defmodule Indexer.Block.Transform.BaseTest do defmodule Indexer.Transform.Blocks.BaseTest do
use ExUnit.Case use ExUnit.Case
alias Indexer.Block.Transform.Base alias Indexer.Transform.Blocks.Base
@block %{ @block %{
difficulty: 1, difficulty: 1,

@ -1,7 +1,7 @@
defmodule Indexer.Block.Transform.CliqueTest do defmodule Indexer.Transform.Blocks.CliqueTest do
use ExUnit.Case use ExUnit.Case
alias Indexer.Block.Transform.Clique alias Indexer.Transform.Blocks.Clique
@block %{ @block %{
difficulty: 1, difficulty: 1,

@ -1,7 +1,7 @@
defmodule Indexer.Block.TransformTest do defmodule Indexer.Transform.BlocksTest do
use ExUnit.Case use ExUnit.Case
alias Indexer.Block.Transform alias Indexer.Transform.Blocks
@block %{ @block %{
difficulty: 1, difficulty: 1,
@ -44,7 +44,42 @@ defmodule Indexer.Block.TransformTest do
end end
test "transforms a list of blocks" do test "transforms a list of blocks" do
assert Transform.transform_blocks(@blocks) assert Blocks.transform_blocks(@blocks)
end end
end end
test "signer/1" do
data = %{
difficulty: 1,
extra_data:
"0xd68301080d846765746886676f312e3130856c696e7578000000000000000000773ab2ca8f47904a14739ad80a75b71d9d29b9fff8b7ecdcb73efffa6f74122f17d304b5dc8e6e5f256c9474dd115c8d4dae31b7a3d409e5c3270f8fde41cd8c00",
gas_limit: 7_753_377,
gas_used: 1_810_195,
hash: "0x7004c895e812c55b0c2be8a46d72ca300a683dc27d1d7917ee7742d4d0359c1f",
logs_bloom:
"0x00000000000000020000000000002000000400000000000000000000000000000000000000000000040000080004000020000010000000000000000000000000000000000000000008000008000000000000000000200000000000000000000000000000020000000000000000000800000000000000804000000010080000000800000000000000000000000000000000000000000000800000000000080000000008000400000000404000000000000000000000000200000000000000000000000002000000000000001002000000000000002000000008000000000020000000000000000000000000000000000000000000000000400000800000000000",
miner_hash: "0x0000000000000000000000000000000000000000",
mix_hash: "0x0000000000000000000000000000000000000000000000000000000000000000",
nonce: "0x0000000000000000",
number: 2_848_394,
parent_hash: "0x20350fc367e19d3865be1ea7da72ab81f8f9941c43ac6bb24a34a0a7caa2f3df",
receipts_root: "0x6ade4ac1079ea50cfadcce2b75ffbe4f9b14bf69b4607bbf1739463076ca6246",
sha3_uncles: "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347",
size: 6437,
state_root: "0x23f63347851bcd109059d007d71e19c4f5e73b7f0862bebcd04458333a004d92",
timestamp: DateTime.from_unix!(1_534_796_040),
total_difficulty: 5_353_647,
transactions: [
"0x7e3bb851fc74a436826d2af6b96e4db9484431811ef0d9c9e78370488d33d4e5",
"0x3976fd1e3d2a715c3cfcfde9bd3210798c26c017b8edb841d319227ecb3322fb",
"0xd8db124005bb8b6fda7b71fd56ac782552a66af58fe843ba3c4930423b87d1d2",
"0x10c1a1ca4d9f4b2bd5b89f7bbcbbc2d69e166fe23662b8db4f6beae0f50ac9fd",
"0xaa58a6545677c796a56b8bc874174c8cfd31a6c6e6ca3a87e086d4f66d52858a"
],
transactions_root: "0xde8d25c0b9b54310128a21601331094b43f910f9f96102869c2e2dca94884bf4",
uncles: []
}
assert Blocks.signer(data) == "0xfc18cbc391de84dbd87db83b20935d3e89f5dd91"
end
end end

@ -1,9 +1,9 @@
defmodule Indexer.MintTransferTest do defmodule Indexer.Transform.MintTransfersTest do
use ExUnit.Case, async: true use ExUnit.Case, async: true
alias Indexer.MintTransfer alias Indexer.Transform.MintTransfers
doctest Indexer.MintTransfer, import: true doctest MintTransfers, import: true
describe "parse/1" do describe "parse/1" do
test "parses logs for fetch the mint transfer" do test "parses logs for fetch the mint transfer" do
@ -32,7 +32,7 @@ defmodule Indexer.MintTransferTest do
] ]
} }
assert MintTransfer.parse(logs) == expected assert MintTransfers.parse(logs) == expected
end end
end end
@ -52,6 +52,6 @@ defmodule Indexer.MintTransferTest do
} }
] ]
assert MintTransfer.parse(logs) == %{mint_transfers: []} assert MintTransfers.parse(logs) == %{mint_transfers: []}
end end
end end

@ -1,9 +1,9 @@
defmodule Indexer.TokenTransfer.ParserTest do defmodule Indexer.Transform.TokenTransfersTest do
use ExUnit.Case use ExUnit.Case
import ExUnit.CaptureLog import ExUnit.CaptureLog
alias Indexer.TokenTransfer.Parser alias Indexer.Transform.TokenTransfers
describe "parse/1" do describe "parse/1" do
test "parse/1 parses logs for tokens and token transfers" do test "parse/1 parses logs for tokens and token transfers" do
@ -82,7 +82,7 @@ defmodule Indexer.TokenTransfer.ParserTest do
] ]
} }
assert Parser.parse(logs) == expected assert TokenTransfers.parse(logs) == expected
end end
test "parses ERC-721 transfer with addresses in data field" do test "parses ERC-721 transfer with addresses in data field" do
@ -121,7 +121,7 @@ defmodule Indexer.TokenTransfer.ParserTest do
] ]
} }
assert Parser.parse([log]) == expected assert TokenTransfers.parse([log]) == expected
end end
test "logs error with unrecognized token transfer format" do test "logs error with unrecognized token transfer format" do
@ -138,7 +138,7 @@ defmodule Indexer.TokenTransfer.ParserTest do
type: "mined" type: "mined"
} }
error = capture_log(fn -> %{tokens: [], token_transfers: []} = Parser.parse([log]) end) error = capture_log(fn -> %{tokens: [], token_transfers: []} = TokenTransfers.parse([log]) end)
assert error =~ ~r"unknown token transfer"i assert error =~ ~r"unknown token transfer"i
end end
end end

@ -1,5 +1,5 @@
defmodule Indexer.Code.Supervisor.Case do defmodule Indexer.Fetcher.BlockReward.Supervisor.Case do
alias Indexer.Code alias Indexer.Fetcher.BlockReward
def start_supervised!(fetcher_arguments \\ []) when is_list(fetcher_arguments) do def start_supervised!(fetcher_arguments \\ []) when is_list(fetcher_arguments) do
merged_fetcher_arguments = merged_fetcher_arguments =
@ -11,7 +11,7 @@ defmodule Indexer.Code.Supervisor.Case do
) )
[merged_fetcher_arguments] [merged_fetcher_arguments]
|> Code.Supervisor.child_spec() |> BlockReward.Supervisor.child_spec()
|> ExUnit.Callbacks.start_supervised!() |> ExUnit.Callbacks.start_supervised!()
end end
end end

@ -1,5 +1,5 @@
defmodule Indexer.CoinBalance.Supervisor.Case do defmodule Indexer.Fetcher.CoinBalance.Supervisor.Case do
alias Indexer.CoinBalance alias Indexer.Fetcher.CoinBalance
def start_supervised!(fetcher_arguments \\ []) when is_list(fetcher_arguments) do def start_supervised!(fetcher_arguments \\ []) when is_list(fetcher_arguments) do
merged_fetcher_arguments = merged_fetcher_arguments =

@ -1,5 +1,5 @@
defmodule Indexer.Block.Reward.Supervisor.Case do defmodule Indexer.Fetcher.ContractCode.Supervisor.Case do
alias Indexer.Block.Reward alias Indexer.Fetcher.ContractCode
def start_supervised!(fetcher_arguments \\ []) when is_list(fetcher_arguments) do def start_supervised!(fetcher_arguments \\ []) when is_list(fetcher_arguments) do
merged_fetcher_arguments = merged_fetcher_arguments =
@ -11,7 +11,7 @@ defmodule Indexer.Block.Reward.Supervisor.Case do
) )
[merged_fetcher_arguments] [merged_fetcher_arguments]
|> Reward.Supervisor.child_spec() |> ContractCode.Supervisor.child_spec()
|> ExUnit.Callbacks.start_supervised!() |> ExUnit.Callbacks.start_supervised!()
end end
end end

@ -1,5 +1,5 @@
defmodule Indexer.InternalTransaction.Supervisor.Case do defmodule Indexer.Fetcher.InternalTransaction.Supervisor.Case do
alias Indexer.InternalTransaction alias Indexer.Fetcher.InternalTransaction
def start_supervised!(fetcher_arguments \\ []) when is_list(fetcher_arguments) do def start_supervised!(fetcher_arguments \\ []) when is_list(fetcher_arguments) do
merged_fetcher_arguments = merged_fetcher_arguments =

@ -1,5 +1,5 @@
defmodule Indexer.PendingTransaction.Supervisor.Case do defmodule Indexer.Fetcher.PendingTransaction.Supervisor.Case do
alias Indexer.PendingTransaction alias Indexer.Fetcher.PendingTransaction
def start_supervised!(fetcher_arguments \\ []) when is_list(fetcher_arguments) do def start_supervised!(fetcher_arguments \\ []) when is_list(fetcher_arguments) do
merged_fetcher_arguments = merged_fetcher_arguments =

@ -1,5 +1,5 @@
defmodule Indexer.ReplacedTransaction.Supervisor.Case do defmodule Indexer.Fetcher.ReplacedTransaction.Supervisor.Case do
alias Indexer.ReplacedTransaction alias Indexer.Fetcher.ReplacedTransaction
def start_supervised!(fetcher_arguments \\ []) when is_list(fetcher_arguments) do def start_supervised!(fetcher_arguments \\ []) when is_list(fetcher_arguments) do
merged_fetcher_arguments = merged_fetcher_arguments =

@ -1,5 +1,5 @@
defmodule Indexer.TokenBalance.Supervisor.Case do defmodule Indexer.Fetcher.TokenBalance.Supervisor.Case do
alias Indexer.TokenBalance alias Indexer.Fetcher.TokenBalance
def start_supervised!(fetcher_arguments \\ []) when is_list(fetcher_arguments) do def start_supervised!(fetcher_arguments \\ []) when is_list(fetcher_arguments) do
merged_fetcher_arguments = merged_fetcher_arguments =

@ -1,5 +1,5 @@
defmodule Indexer.Token.Supervisor.Case do defmodule Indexer.Fetcher.Token.Supervisor.Case do
alias Indexer.Token alias Indexer.Fetcher.Token
def start_supervised!(fetcher_arguments \\ []) when is_list(fetcher_arguments) do def start_supervised!(fetcher_arguments \\ []) when is_list(fetcher_arguments) do
merged_fetcher_arguments = merged_fetcher_arguments =

@ -1,5 +1,5 @@
defmodule Indexer.Block.Uncle.Supervisor.Case do defmodule Indexer.Fetcher.UncleBlock.Supervisor.Case do
alias Indexer.Block alias Indexer.Fetcher.UncleBlock
def start_supervised!(fetcher_arguments \\ []) when is_list(fetcher_arguments) do def start_supervised!(fetcher_arguments \\ []) when is_list(fetcher_arguments) do
merged_fetcher_arguments = merged_fetcher_arguments =
@ -11,7 +11,7 @@ defmodule Indexer.Block.Uncle.Supervisor.Case do
) )
[merged_fetcher_arguments] [merged_fetcher_arguments]
|> Block.Uncle.Supervisor.child_spec() |> UncleBlock.Supervisor.child_spec()
|> ExUnit.Callbacks.start_supervised!() |> ExUnit.Callbacks.start_supervised!()
end end
end end
Loading…
Cancel
Save