|
|
|
@ -1,10 +1,6 @@ |
|
|
|
|
defmodule Explorer.Indexer.BlockFetcher do |
|
|
|
|
@moduledoc """ |
|
|
|
|
TODO |
|
|
|
|
|
|
|
|
|
## Next steps |
|
|
|
|
|
|
|
|
|
- after gensis index transition to RT index |
|
|
|
|
Fetches and indexes block ranges from gensis to realtime. |
|
|
|
|
""" |
|
|
|
|
|
|
|
|
|
use GenServer |
|
|
|
@ -15,39 +11,50 @@ defmodule Explorer.Indexer.BlockFetcher do |
|
|
|
|
|
|
|
|
|
alias Explorer.Indexer.{ |
|
|
|
|
Sequence, |
|
|
|
|
AddressFetcher |
|
|
|
|
BlockImporter, |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
alias Explorer.JSONRPC.Transactions |
|
|
|
|
|
|
|
|
|
# Struct |
|
|
|
|
|
|
|
|
|
defstruct ~w(current_block genesis_task subscription_id)a |
|
|
|
|
|
|
|
|
|
# Constants |
|
|
|
|
|
|
|
|
|
@batch_size 50 |
|
|
|
|
@blocks_concurrency 20 |
|
|
|
|
@blocks_concurrency 10 |
|
|
|
|
|
|
|
|
|
@internal_batch_size 50 |
|
|
|
|
@internal_concurrency 8 |
|
|
|
|
|
|
|
|
|
@polling_interval 20_000 |
|
|
|
|
@block_rate 5_000 |
|
|
|
|
|
|
|
|
|
@receipts_batch_size 250 |
|
|
|
|
@receipts_concurrency 20 |
|
|
|
|
|
|
|
|
|
# Functions |
|
|
|
|
|
|
|
|
|
@doc """ |
|
|
|
|
Starts the server. |
|
|
|
|
|
|
|
|
|
## Options |
|
|
|
|
|
|
|
|
|
* `:block_rate` - The millisecond rate new blocks are published at. |
|
|
|
|
Defaults to `#{@block_rate}`. |
|
|
|
|
""" |
|
|
|
|
def start_link(opts) do |
|
|
|
|
GenServer.start_link(__MODULE__, opts, name: __MODULE__) |
|
|
|
|
end |
|
|
|
|
|
|
|
|
|
## GenServer callbacks |
|
|
|
|
|
|
|
|
|
@impl GenServer |
|
|
|
|
def init(opts) do |
|
|
|
|
send(self(), :catchup_index) |
|
|
|
|
:timer.send_interval(15_000, self(), :debug_count) |
|
|
|
|
|
|
|
|
|
def handle_info(:index, state) do |
|
|
|
|
state = %{ |
|
|
|
|
genesis_task: nil, |
|
|
|
|
realtime_interval: (opts[:block_rate] || @block_rate) * 2, |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
{:ok, state} |
|
|
|
|
end |
|
|
|
|
|
|
|
|
|
@impl GenServer |
|
|
|
|
def handle_info(:catchup_index, state) do |
|
|
|
|
{count, missing_ranges} = missing_block_numbers() |
|
|
|
|
current_block = Indexer.next_block_number() |
|
|
|
|
|
|
|
|
@ -55,35 +62,34 @@ defmodule Explorer.Indexer.BlockFetcher do |
|
|
|
|
|
|
|
|
|
{:ok, genesis_task} = |
|
|
|
|
Task.start_link(fn -> |
|
|
|
|
stream_import(missing_ranges, current_block) |
|
|
|
|
{:ok, seq} = Sequence.start_link(missing_ranges, current_block, @batch_size) |
|
|
|
|
stream_import(seq, max_concurrency: @blocks_concurrency) |
|
|
|
|
end) |
|
|
|
|
|
|
|
|
|
Process.monitor(genesis_task) |
|
|
|
|
|
|
|
|
|
{:noreply, %__MODULE__{state | genesis_task: genesis_task}} |
|
|
|
|
{:noreply, %{state | genesis_task: genesis_task}} |
|
|
|
|
end |
|
|
|
|
|
|
|
|
|
def handle_info(:poll, %__MODULE__{subscription_id: subscription_id} = state) do |
|
|
|
|
Process.send_after(self(), :poll, @polling_interval) |
|
|
|
|
|
|
|
|
|
with {:ok, blocks} when length(blocks) > 0 <- JSONRPC.check_for_updates(subscription_id) do |
|
|
|
|
Logger.debug(fn -> "Processing #{length(blocks)} new block(s)" end) |
|
|
|
|
def handle_info(:realtime_index, state) do |
|
|
|
|
{:ok, realtime_task} = |
|
|
|
|
Task.start_link(fn -> |
|
|
|
|
{:ok, seq} = Sequence.start_link([], Indexer.next_block_number(), 2) |
|
|
|
|
stream_import(seq, max_concurrency: 1) |
|
|
|
|
end) |
|
|
|
|
|
|
|
|
|
# TODO do something with the new blocks |
|
|
|
|
JSONRPC.fetch_blocks_by_hash(blocks) |
|
|
|
|
end |
|
|
|
|
Process.monitor(realtime_task) |
|
|
|
|
|
|
|
|
|
{:noreply, state} |
|
|
|
|
{:noreply, %{state | realtime_task: realtime_task}} |
|
|
|
|
end |
|
|
|
|
|
|
|
|
|
def handle_info({:DOWN, _ref, :process, pid, :normal}, %__MODULE__{genesis_task: pid} = state) do |
|
|
|
|
Logger.info(fn -> "Finished index from genesis" end) |
|
|
|
|
|
|
|
|
|
{:ok, subscription_id} = JSONRPC.listen_for_new_blocks() |
|
|
|
|
|
|
|
|
|
send(self(), :poll) |
|
|
|
|
def handle_info({:DOWN, _ref, :process, pid, :normal}, %{realtime_task: pid} = state) do |
|
|
|
|
{:noreply, schedule_next_realtime_fetch(%{state | realtime_task: nil})} |
|
|
|
|
end |
|
|
|
|
|
|
|
|
|
{:noreply, %__MODULE__{state | genesis_task: nil, subscription_id: subscription_id}} |
|
|
|
|
def handle_info({:DOWN, _ref, :process, pid, :normal}, %{genesis_task: pid} = state) do |
|
|
|
|
Logger.info(fn -> "Finished index from genesis. Transitioning to realtime index." end) |
|
|
|
|
{:noreply, schedule_next_realtime_fetch(%{state | genesis_task: nil})} |
|
|
|
|
end |
|
|
|
|
|
|
|
|
|
def handle_info(:debug_count, state) do |
|
|
|
@ -104,18 +110,7 @@ defmodule Explorer.Indexer.BlockFetcher do |
|
|
|
|
{:noreply, state} |
|
|
|
|
end |
|
|
|
|
|
|
|
|
|
@impl GenServer |
|
|
|
|
def init(_opts) do |
|
|
|
|
send(self(), :index) |
|
|
|
|
:timer.send_interval(15_000, self(), :debug_count) |
|
|
|
|
|
|
|
|
|
{:ok, %__MODULE__{current_block: 0, genesis_task: nil, subscription_id: nil}} |
|
|
|
|
end |
|
|
|
|
|
|
|
|
|
## Private Functions |
|
|
|
|
|
|
|
|
|
defp cap_seq(seq, :end_of_chain, {_block_start, block_end}) do |
|
|
|
|
Logger.info("Reached end of blockchain #{inspect(block_end)}") |
|
|
|
|
defp cap_seq(seq, :end_of_chain, {_block_start, _block_end}) do |
|
|
|
|
:ok = Sequence.cap(seq) |
|
|
|
|
end |
|
|
|
|
|
|
|
|
@ -134,13 +129,13 @@ defmodule Explorer.Indexer.BlockFetcher do |
|
|
|
|
|> Enum.chunk_every(@internal_batch_size) |
|
|
|
|
|> Task.async_stream(&JSONRPC.fetch_internal_transactions(&1), stream_opts) |
|
|
|
|
|> Enum.reduce_while({:ok, []}, fn |
|
|
|
|
{:ok, {:ok, internal_transactions_params}}, {:ok, acc} -> {:cont, {:ok, acc ++ internal_transactions_params}} |
|
|
|
|
{:ok, {:ok, internal_transactions}}, {:ok, acc} -> {:cont, {:ok, acc ++ internal_transactions}} |
|
|
|
|
{:ok, {:error, reason}}, {:ok, _acc} -> {:halt, {:error, reason}} |
|
|
|
|
{:error, reason}, {:ok, _acc} -> {:halt, {:error, reason}} |
|
|
|
|
end) |
|
|
|
|
end |
|
|
|
|
|
|
|
|
|
defp fetch_transaction_receipts([]), do: {:ok, %{logs_params: [], receipts_params: []}} |
|
|
|
|
defp fetch_transaction_receipts([]), do: {:ok, %{logs: [], receipts: []}} |
|
|
|
|
|
|
|
|
|
defp fetch_transaction_receipts(hashes) do |
|
|
|
|
Logger.debug(fn -> "fetching #{length(hashes)} transaction receipts" end) |
|
|
|
@ -149,11 +144,10 @@ defmodule Explorer.Indexer.BlockFetcher do |
|
|
|
|
hashes |
|
|
|
|
|> Enum.chunk_every(@receipts_batch_size) |
|
|
|
|
|> Task.async_stream(&JSONRPC.fetch_transaction_receipts(&1), stream_opts) |
|
|
|
|
|> Enum.reduce_while({:ok, %{logs_params: [], receipts_params: []}}, fn |
|
|
|
|
{:ok, {:ok, %{logs_params: logs_params, receipts_params: receipts_params}}}, |
|
|
|
|
{:ok, %{logs_params: acc_log_params, receipts_params: acc_receipts_params}} -> |
|
|
|
|
{:cont, |
|
|
|
|
{:ok, %{logs_params: acc_log_params ++ logs_params, receipts_params: acc_receipts_params ++ receipts_params}}} |
|
|
|
|
|> Enum.reduce_while({:ok, %{logs: [], receipts: []}}, fn |
|
|
|
|
{:ok, {:ok, %{logs: logs, receipts: receipts}}}, |
|
|
|
|
{:ok, %{logs: acc_logs, receipts: acc_receipts}} -> |
|
|
|
|
{:cont, {:ok, %{logs: acc_logs ++ logs, receipts: acc_receipts ++ receipts}}} |
|
|
|
|
|
|
|
|
|
{:ok, {:error, reason}}, {:ok, _acc} -> |
|
|
|
|
{:halt, {:error, reason}} |
|
|
|
@ -163,27 +157,12 @@ defmodule Explorer.Indexer.BlockFetcher do |
|
|
|
|
end) |
|
|
|
|
end |
|
|
|
|
|
|
|
|
|
defp insert(%{ |
|
|
|
|
blocks_params: blocks_params, |
|
|
|
|
internal_transactions_params: internal_transactions_params, |
|
|
|
|
logs_params: log_params, |
|
|
|
|
range: range, |
|
|
|
|
receipts_params: receipt_params, |
|
|
|
|
seq: seq, |
|
|
|
|
transactions_params: transactions_params |
|
|
|
|
}) do |
|
|
|
|
case Chain.import_blocks(%{ |
|
|
|
|
blocks_params: blocks_params, |
|
|
|
|
internal_transactions_params: internal_transactions_params, |
|
|
|
|
logs_params: log_params, |
|
|
|
|
receipts_params: receipt_params, |
|
|
|
|
transactions_params: transactions_params |
|
|
|
|
}) do |
|
|
|
|
{:ok, %{addresses: address_hashes}} -> |
|
|
|
|
:ok = AddressFetcher.async_fetch_balances(address_hashes) |
|
|
|
|
defp insert(seq, range, params) do |
|
|
|
|
case BlockImporter.import_blocks(params) do |
|
|
|
|
:ok -> |
|
|
|
|
:ok |
|
|
|
|
|
|
|
|
|
{:error, step, reason, _changes} -> |
|
|
|
|
{:error, step, reason} -> |
|
|
|
|
Logger.debug(fn -> |
|
|
|
|
"failed to insert blocks during #{step} #{inspect(range)}: #{inspect(reason)}. Retrying" |
|
|
|
|
end) |
|
|
|
@ -216,50 +195,43 @@ defmodule Explorer.Indexer.BlockFetcher do |
|
|
|
|
{count, chunked_ranges} |
|
|
|
|
end |
|
|
|
|
|
|
|
|
|
defp stream_import(missing_ranges, current_block) do |
|
|
|
|
{:ok, seq} = Sequence.start_link(missing_ranges, current_block, @batch_size) |
|
|
|
|
|
|
|
|
|
defp stream_import(seq, task_opts) do |
|
|
|
|
seq |
|
|
|
|
|> Sequence.build_stream() |
|
|
|
|
|> Task.async_stream( |
|
|
|
|
fn {block_start, block_end} = range -> |
|
|
|
|
with {:ok, value} <- JSONRPC.fetch_blocks_by_range(block_start, block_end), |
|
|
|
|
# `mix format` bug made the line too long when pattern combined into above line |
|
|
|
|
%{next: next, blocks_params: blocks_params, range: range, transactions_params: transactions_params} = |
|
|
|
|
value, |
|
|
|
|
with {:ok, next, result} <- JSONRPC.fetch_blocks_by_range(block_start, block_end), |
|
|
|
|
%{blocks: blocks, transactions: transactions} <- result, |
|
|
|
|
:ok <- cap_seq(seq, next, range), |
|
|
|
|
transaction_hashes <- Transactions.params_to_hashes(transactions_params), |
|
|
|
|
{:ok, %{logs_params: logs_params, receipts_params: receipts_params}} <- |
|
|
|
|
fetch_transaction_receipts(transaction_hashes), |
|
|
|
|
{:ok, internal_transactions_params} <- fetch_internal_transactions(transaction_hashes) do |
|
|
|
|
insert(%{ |
|
|
|
|
blocks_params: blocks_params, |
|
|
|
|
internal_transactions_params: internal_transactions_params, |
|
|
|
|
logs_params: logs_params, |
|
|
|
|
range: range, |
|
|
|
|
receipts_params: receipts_params, |
|
|
|
|
seq: seq, |
|
|
|
|
transactions_params: transactions_params |
|
|
|
|
transaction_hashes <- Transactions.params_to_hashes(transactions), |
|
|
|
|
{:ok, receipt_params} <- fetch_transaction_receipts(transaction_hashes), |
|
|
|
|
%{logs: logs, receipts: receipts} <- receipt_params, |
|
|
|
|
{:ok, internal_transactions} <- fetch_internal_transactions(transaction_hashes) do |
|
|
|
|
|
|
|
|
|
insert(seq, range, %{ |
|
|
|
|
blocks: blocks, |
|
|
|
|
internal_transactions: internal_transactions, |
|
|
|
|
logs: logs, |
|
|
|
|
receipts: receipts, |
|
|
|
|
transactions: transactions |
|
|
|
|
}) |
|
|
|
|
|
|
|
|
|
else |
|
|
|
|
{:error, reason} -> |
|
|
|
|
Logger.debug(fn -> |
|
|
|
|
"failed to fetch blocks #{inspect(range)}: #{inspect(reason)}. Retrying" |
|
|
|
|
end) |
|
|
|
|
|
|
|
|
|
:ok = Sequence.inject_range(seq, range) |
|
|
|
|
|
|
|
|
|
{:error, reason, range} -> |
|
|
|
|
Logger.debug(fn -> |
|
|
|
|
"failed to fetch blocks #{inspect(range)}: #{inspect(reason)}. Retrying" |
|
|
|
|
end) |
|
|
|
|
|
|
|
|
|
:ok = Sequence.inject_range(seq, range) |
|
|
|
|
end |
|
|
|
|
end, |
|
|
|
|
max_concurrency: @blocks_concurrency, |
|
|
|
|
timeout: :infinity |
|
|
|
|
Keyword.merge(task_opts, timeout: :infinity) |
|
|
|
|
) |
|
|
|
|
|> Enum.each(fn {:ok, :ok} -> :ok end) |
|
|
|
|
end |
|
|
|
|
|
|
|
|
|
defp schedule_next_realtime_fetch(state) do |
|
|
|
|
timer = Process.send_after(self(), :realtime_index, state.realtime_interval) |
|
|
|
|
%{state | poll_timer: timer} |
|
|
|
|
end |
|
|
|
|
end |
|
|
|
|