Separate Catchup and Realtime indexer's fetching, so that Catchup
continues to use the old polling, `EthereumJSONRPC.json_rpc` while Realtime
subscribes with `EthereumJSONRPC.subscribe`.
Because Realtime gets notified of new blocks through the subscription,
it no longer has a Sequence, so Sequence has been removed from
`%BlockFetcher{}`. There is also no block concurrency or block batch
size for Realtime, so that was removed from `%BlockFetcher{}` too and
moved to `%BlockFetcher.Catchup{}`.
Because Realtime requires a websocket to be connected, there is now an
`Indexer.BlockFetcher.Realtime.Supervisor` that supervises the
`Indexer.BlockFetcher.Realtime.WebSocket` and
`Indexer.BlockFetcher.Realtime`. `Indexer.BlockFetcher.Realtime`
because it is waiting on notifications from is subscription becomes a
`GenServer` insteado of a `Task`, so it can be supervised normally.
BlockFetcher.Supervisor becomes a normal `Supervisor` of
`Catchup.Supervisor` and `Realtime.Supervisor`.
As Realtime is using `EthereumJSONRPC.subscribe`,
`subscribe_named_arguments` has been added as a sibling of
`json_rpc_named_arguments` in the `:indexer` `config`s.
pull/572/head
Luke Imhoff
6 years ago
committed by
Luke Imhoff
parent
ae21b7e8b0
commit
c8c53405fc
@ -0,0 +1,110 @@ |
||||
defmodule Indexer.BlockFetcher.Catchup.Supervisor do |
||||
@moduledoc """ |
||||
Supervises the `Indexer.BlockerFetcher.Catchup` with exponential backoff for restarts. |
||||
""" |
||||
|
||||
# NOT a `Supervisor` because of the `Task` restart strategies are custom. |
||||
use GenServer |
||||
|
||||
require Logger |
||||
|
||||
alias Indexer.{BlockFetcher, BoundInterval} |
||||
alias Indexer.BlockFetcher.Catchup |
||||
|
||||
# milliseconds |
||||
@block_interval 5_000 |
||||
|
||||
@enforce_keys ~w(bound_interval catchup)a |
||||
defstruct bound_interval: nil, |
||||
catchup: %Catchup{}, |
||||
task: nil |
||||
|
||||
def child_spec(arg) do |
||||
# The `child_spec` from `use Supervisor` because the one from `use GenServer` will set the `type` to `:worker` |
||||
# instead of `:supervisor` and use the wrong shutdown timeout |
||||
Supervisor.child_spec(%{id: __MODULE__, start: {__MODULE__, :start_link, [arg]}, type: :supervisor}, []) |
||||
end |
||||
|
||||
@doc """ |
||||
Starts supervisor of `Indexer.BlockerFetcher.Catchup` and `Indexer.BlockFetcher.Realtime`. |
||||
|
||||
For `named_arguments` see `Indexer.BlockFetcher.new/1`. For `t:GenServer.options/0` see `GenServer.start_link/3`. |
||||
""" |
||||
@spec start_link([named_arguments :: list() | GenServer.options()]) :: {:ok, pid} |
||||
def start_link([named_arguments, gen_server_options]) when is_map(named_arguments) and is_list(gen_server_options) do |
||||
GenServer.start_link(__MODULE__, named_arguments, gen_server_options) |
||||
end |
||||
|
||||
@impl GenServer |
||||
def init(named_arguments) do |
||||
state = new(named_arguments) |
||||
|
||||
send(self(), :catchup_index) |
||||
|
||||
{:ok, state} |
||||
end |
||||
|
||||
defp new(%{block_fetcher: common_block_fetcher} = named_arguments) do |
||||
block_fetcher = %BlockFetcher{common_block_fetcher | broadcast: false, callback_module: Catchup} |
||||
|
||||
block_interval = Map.get(named_arguments, :block_interval, @block_interval) |
||||
minimum_interval = div(block_interval, 2) |
||||
bound_interval = BoundInterval.within(minimum_interval..(minimum_interval * 10)) |
||||
|
||||
%__MODULE__{ |
||||
catchup: %Catchup{block_fetcher: block_fetcher}, |
||||
bound_interval: bound_interval |
||||
} |
||||
end |
||||
|
||||
@impl GenServer |
||||
def handle_info(:catchup_index, %__MODULE__{catchup: %Catchup{} = catchup} = state) do |
||||
{:noreply, |
||||
%__MODULE__{state | task: Task.Supervisor.async_nolink(Indexer.TaskSupervisor, Catchup, :task, [catchup])}} |
||||
end |
||||
|
||||
def handle_info( |
||||
{ref, %{first_block_number: first_block_number, missing_block_count: missing_block_count}}, |
||||
%__MODULE__{ |
||||
bound_interval: bound_interval, |
||||
task: %Task{ref: ref} |
||||
} = state |
||||
) |
||||
when is_integer(missing_block_count) do |
||||
new_bound_interval = |
||||
case missing_block_count do |
||||
0 -> |
||||
Logger.info("Index already caught up in #{first_block_number}-0") |
||||
|
||||
BoundInterval.increase(bound_interval) |
||||
|
||||
_ -> |
||||
Logger.info("Index had to catch up #{missing_block_count} blocks in #{first_block_number}-0") |
||||
|
||||
BoundInterval.decrease(bound_interval) |
||||
end |
||||
|
||||
Process.demonitor(ref, [:flush]) |
||||
|
||||
interval = new_bound_interval.current |
||||
|
||||
Logger.info(fn -> |
||||
"Checking if index needs to catch up in #{interval}ms" |
||||
end) |
||||
|
||||
Process.send_after(self(), :catchup_index, interval) |
||||
|
||||
{:noreply, %__MODULE__{state | bound_interval: new_bound_interval, task: nil}} |
||||
end |
||||
|
||||
def handle_info( |
||||
{:DOWN, ref, :process, pid, reason}, |
||||
%__MODULE__{task: %Task{pid: pid, ref: ref}} = state |
||||
) do |
||||
Logger.error(fn -> "Catchup index stream exited with reason (#{inspect(reason)}). Restarting" end) |
||||
|
||||
send(self(), :catchup_index) |
||||
|
||||
{:noreply, %__MODULE__{state | task: nil}} |
||||
end |
||||
end |
||||
@ -0,0 +1,37 @@ |
||||
defmodule Indexer.BlockFetcher.Realtime.Supervisor do |
||||
@moduledoc """ |
||||
Supervises realtime block fetcher. |
||||
""" |
||||
|
||||
use Supervisor |
||||
|
||||
def start_link([arguments, gen_server_options]) do |
||||
Supervisor.start_link(__MODULE__, arguments, gen_server_options) |
||||
end |
||||
|
||||
@impl Supervisor |
||||
def init(%{block_fetcher: block_fetcher, subscribe_named_arguments: subscribe_named_arguments}) do |
||||
children = |
||||
case Keyword.fetch!(subscribe_named_arguments, :transport) do |
||||
EthereumJSONRPC.WebSocket -> |
||||
transport_options = Keyword.fetch!(subscribe_named_arguments, :transport_options) |
||||
url = Keyword.fetch!(transport_options, :url) |
||||
web_socket_module = Keyword.fetch!(transport_options, :web_socket) |
||||
web_socket = Indexer.BlockFetcher.Realtime.WebSocket |
||||
|
||||
block_fetcher_subscribe_named_arguments = |
||||
put_in(subscribe_named_arguments[:transport_options][:web_socket_options], %{web_socket: web_socket}) |
||||
|
||||
[ |
||||
{web_socket_module, [url, [name: web_socket]]}, |
||||
{Indexer.BlockFetcher.Realtime, |
||||
[ |
||||
%{block_fetcher: block_fetcher, subscribe_named_arguments: block_fetcher_subscribe_named_arguments}, |
||||
[name: Indexer.BlockFetcher.Realtime] |
||||
]} |
||||
] |
||||
end |
||||
|
||||
Supervisor.init(children, strategy: :rest_for_one) |
||||
end |
||||
end |
||||
@ -1,84 +1,35 @@ |
||||
defmodule Indexer.BlockFetcher.Supervisor do |
||||
@moduledoc """ |
||||
Supervises the `Indexer.BlockerFetcher.Catchup` and `Indexer.BlockFetcher.Realtime`. |
||||
Supervises catchup and realtime block fetchers |
||||
""" |
||||
|
||||
# NOT a `Supervisor` because of the `Task` restart strategies are custom. |
||||
use GenServer |
||||
|
||||
require Logger |
||||
|
||||
alias Indexer.BlockFetcher |
||||
alias Indexer.BlockFetcher.{Catchup, Realtime} |
||||
|
||||
# milliseconds |
||||
@block_interval 5_000 |
||||
|
||||
@enforce_keys ~w(catchup realtime)a |
||||
defstruct ~w(catchup realtime)a |
||||
|
||||
def child_spec(arg) do |
||||
# The `child_spec` from `use Supervisor` because the one from `use GenServer` will set the `type` to `:worker` |
||||
# instead of `:supervisor` and use the wrong shutdown timeout |
||||
Supervisor.child_spec(%{id: __MODULE__, start: {__MODULE__, :start_link, [arg]}, type: :supervisor}, []) |
||||
end |
||||
|
||||
@doc """ |
||||
Starts supervisor of `Indexer.BlockerFetcher.Catchup` and `Indexer.BlockFetcher.Realtime`. |
||||
|
||||
For `named_arguments` see `Indexer.BlockFetcher.new/1`. For `t:GenServer.options/0` see `GenServer.start_link/3`. |
||||
""" |
||||
@spec start_link([named_arguments :: list() | GenServer.options()]) :: {:ok, pid} |
||||
def start_link([named_arguments, gen_server_options]) when is_list(named_arguments) and is_list(gen_server_options) do |
||||
GenServer.start_link(__MODULE__, named_arguments, gen_server_options) |
||||
end |
||||
|
||||
@impl GenServer |
||||
def init(named_arguments) do |
||||
state = new(named_arguments) |
||||
|
||||
send(self(), :catchup_index) |
||||
{:ok, _} = :timer.send_interval(state.realtime.interval, :realtime_index) |
||||
|
||||
{:ok, state} |
||||
end |
||||
|
||||
defp new(named_arguments) do |
||||
{given_block_interval, block_fetcher_named_arguments} = Keyword.pop(named_arguments, :block_interval) |
||||
block_fetcher = struct!(BlockFetcher, block_fetcher_named_arguments) |
||||
block_interval = given_block_interval || @block_interval |
||||
|
||||
%__MODULE__{ |
||||
catchup: Catchup.new(%{block_fetcher: block_fetcher, block_interval: block_interval}), |
||||
realtime: Realtime.new(%{block_fetcher: block_fetcher, block_interval: block_interval}) |
||||
} |
||||
end |
||||
|
||||
@impl GenServer |
||||
def handle_info(:catchup_index, %__MODULE__{} = state) do |
||||
{:noreply, Catchup.put(state)} |
||||
end |
||||
|
||||
def handle_info({ref, _} = message, %__MODULE__{catchup: %Catchup{task: %Task{ref: ref}}} = state) do |
||||
{:noreply, Catchup.handle_success(message, state)} |
||||
end |
||||
|
||||
def handle_info( |
||||
{:DOWN, ref, :process, pid, _} = message, |
||||
%__MODULE__{catchup: %Catchup{task: %Task{pid: pid, ref: ref}}} = state |
||||
) do |
||||
{:noreply, Catchup.handle_failure(message, state)} |
||||
end |
||||
|
||||
def handle_info(:realtime_index, %__MODULE__{} = state) do |
||||
{:noreply, Realtime.put(state)} |
||||
end |
||||
|
||||
def handle_info({ref, :ok} = message, %__MODULE__{} = state) when is_reference(ref) do |
||||
{:noreply, Realtime.handle_success(message, state)} |
||||
end |
||||
|
||||
def handle_info({:DOWN, _, :process, _, _} = message, %__MODULE__{} = state) do |
||||
{:noreply, Realtime.handle_failure(message, state)} |
||||
use Supervisor |
||||
|
||||
def start_link([arguments, gen_server_options]) do |
||||
Supervisor.start_link(__MODULE__, arguments, gen_server_options) |
||||
end |
||||
|
||||
@impl Supervisor |
||||
def init(%{block_interval: block_interval, subscribe_named_arguments: subscribe_named_arguments} = named_arguments) do |
||||
block_fetcher = |
||||
named_arguments |
||||
|> Map.drop(~w(block_interval subscribe_named_arguments)a) |
||||
|> BlockFetcher.new() |
||||
|
||||
Supervisor.init( |
||||
[ |
||||
{Catchup.Supervisor, |
||||
[%{block_fetcher: block_fetcher, block_interval: block_interval}, [name: Catchup.Supervisor]]}, |
||||
{Realtime.Supervisor, |
||||
[ |
||||
%{block_fetcher: block_fetcher, subscribe_named_arguments: subscribe_named_arguments}, |
||||
[name: Realtime.Supervisor] |
||||
]} |
||||
], |
||||
strategy: :one_for_one |
||||
) |
||||
end |
||||
end |
||||
|
||||
Loading…
Reference in new issue