Qwerty5Uiop
2 years ago
parent
b1e7258cf0
commit
cc67f502da
@ -0,0 +1,39 @@ |
|||||||
|
defmodule Indexer.Block.Catchup.Helper do |
||||||
|
@moduledoc """ |
||||||
|
Catchup helper functions |
||||||
|
""" |
||||||
|
|
||||||
|
def sanitize_ranges(ranges) do |
||||||
|
ranges |
||||||
|
|> Enum.filter(&(not is_nil(&1))) |
||||||
|
|> Enum.sort_by( |
||||||
|
fn |
||||||
|
from.._to -> from |
||||||
|
el -> el |
||||||
|
end, |
||||||
|
:asc |
||||||
|
) |
||||||
|
|> Enum.chunk_while( |
||||||
|
nil, |
||||||
|
fn |
||||||
|
_from.._to = chunk, nil -> |
||||||
|
{:cont, chunk} |
||||||
|
|
||||||
|
_ch_from..ch_to = chunk, acc_from..acc_to = acc -> |
||||||
|
if Range.disjoint?(chunk, acc), |
||||||
|
do: {:cont, acc, chunk}, |
||||||
|
else: {:cont, acc_from..max(ch_to, acc_to)} |
||||||
|
|
||||||
|
num, nil -> |
||||||
|
{:halt, num} |
||||||
|
|
||||||
|
num, acc_from.._ = acc -> |
||||||
|
if Range.disjoint?(num..num, acc), do: {:cont, acc, num}, else: {:halt, acc_from} |
||||||
|
|
||||||
|
_, num -> |
||||||
|
{:halt, num} |
||||||
|
end, |
||||||
|
fn reminder -> {:cont, reminder, nil} end |
||||||
|
) |
||||||
|
end |
||||||
|
end |
||||||
@ -0,0 +1,198 @@ |
|||||||
|
defmodule Indexer.Block.Catchup.MissingRangesCollector do |
||||||
|
@moduledoc """ |
||||||
|
Collects missing block ranges. |
||||||
|
""" |
||||||
|
|
||||||
|
use GenServer |
||||||
|
|
||||||
|
alias Explorer.Chain |
||||||
|
alias Explorer.Chain.Cache.BlockNumber |
||||||
|
alias Indexer.Block.Catchup.Helper |
||||||
|
|
||||||
|
@default_missing_ranges_batch_size 100_000 |
||||||
|
@future_check_interval Application.compile_env(:indexer, __MODULE__)[:future_check_interval] |
||||||
|
@past_check_interval 10 |
||||||
|
|
||||||
|
@spec start_link(term()) :: GenServer.on_start() |
||||||
|
def start_link(_) do |
||||||
|
GenServer.start_link(__MODULE__, :ok, name: __MODULE__) |
||||||
|
end |
||||||
|
|
||||||
|
@impl true |
||||||
|
def init(_) do |
||||||
|
{:ok, define_init()} |
||||||
|
end |
||||||
|
|
||||||
|
defp define_init do |
||||||
|
case Application.get_env(:indexer, :block_ranges) do |
||||||
|
nil -> |
||||||
|
default_init() |
||||||
|
|
||||||
|
string_ranges -> |
||||||
|
case parse_block_ranges(string_ranges) do |
||||||
|
:no_ranges -> default_init() |
||||||
|
{:finite_ranges, ranges} -> ranges_init(ranges) |
||||||
|
{:infinite_ranges, ranges, max_fetched_block_number} -> ranges_init(ranges, max_fetched_block_number) |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
defp default_init do |
||||||
|
max_number = BlockNumber.get_max() |
||||||
|
{min_number, first_batch} = fetch_missing_ranges_batch(max_number, false) |
||||||
|
initial_queue = push_batch_to_queue(first_batch, :queue.new()) |
||||||
|
|
||||||
|
Process.send_after(self(), :update_future, @future_check_interval) |
||||||
|
Process.send_after(self(), :update_past, @past_check_interval) |
||||||
|
|
||||||
|
%{queue: initial_queue, min_fetched_block_number: min_number, max_fetched_block_number: max_number} |
||||||
|
end |
||||||
|
|
||||||
|
defp ranges_init(ranges, max_fetched_block_number \\ nil) do |
||||||
|
missing_ranges = |
||||||
|
ranges |
||||||
|
|> Enum.reverse() |
||||||
|
|> Enum.flat_map(fn f..l -> Chain.missing_block_number_ranges(l..f) end) |
||||||
|
|
||||||
|
initial_queue = push_batch_to_queue(missing_ranges, :queue.new()) |
||||||
|
|
||||||
|
if not is_nil(max_fetched_block_number) do |
||||||
|
Process.send_after(self(), :update_future, @future_check_interval) |
||||||
|
end |
||||||
|
|
||||||
|
%{queue: initial_queue, max_fetched_block_number: max_fetched_block_number} |
||||||
|
end |
||||||
|
|
||||||
|
def get_latest_batch do |
||||||
|
GenServer.call(__MODULE__, :get_latest_batch) |
||||||
|
end |
||||||
|
|
||||||
|
@impl true |
||||||
|
def handle_call(:get_latest_batch, _from, %{queue: queue} = state) do |
||||||
|
{latest_batch, new_queue} = |
||||||
|
case :queue.out(queue) do |
||||||
|
{{:value, batch}, rest} -> {batch, rest} |
||||||
|
{:empty, rest} -> {[], rest} |
||||||
|
end |
||||||
|
|
||||||
|
{:reply, latest_batch, %{state | queue: new_queue}} |
||||||
|
end |
||||||
|
|
||||||
|
@impl true |
||||||
|
def handle_info(:update_future, %{queue: queue, max_fetched_block_number: max_number} = state) do |
||||||
|
if continue_future_updating?(max_number) do |
||||||
|
{new_max_number, batch} = fetch_missing_ranges_batch(max_number, true) |
||||||
|
Process.send_after(self(), :update_future, @future_check_interval) |
||||||
|
{:noreply, %{state | queue: push_batch_to_queue(batch, queue, true), max_fetched_block_number: new_max_number}} |
||||||
|
else |
||||||
|
{:noreply, state} |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
def handle_info(:update_past, %{queue: queue, min_fetched_block_number: min_number} = state) do |
||||||
|
if min_number > first_block() do |
||||||
|
{new_min_number, batch} = fetch_missing_ranges_batch(min_number, false) |
||||||
|
Process.send_after(self(), :update_past, @past_check_interval) |
||||||
|
{:noreply, %{state | queue: push_batch_to_queue(batch, queue), min_fetched_block_number: new_min_number}} |
||||||
|
else |
||||||
|
{:noreply, state} |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
defp fetch_missing_ranges_batch(min_fetched_block_number, false = _to_future?) do |
||||||
|
from = min_fetched_block_number - 1 |
||||||
|
to = max(min_fetched_block_number - missing_ranges_batch_size(), first_block()) |
||||||
|
|
||||||
|
if from >= to do |
||||||
|
{to, Chain.missing_block_number_ranges(from..to)} |
||||||
|
else |
||||||
|
{min_fetched_block_number, []} |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
defp fetch_missing_ranges_batch(max_fetched_block_number, true) do |
||||||
|
to = max_fetched_block_number + 1 |
||||||
|
from = min(max_fetched_block_number + missing_ranges_batch_size(), BlockNumber.get_max()) |
||||||
|
|
||||||
|
if from >= to do |
||||||
|
{from, Chain.missing_block_number_ranges(from..to)} |
||||||
|
else |
||||||
|
{max_fetched_block_number, []} |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
defp push_batch_to_queue(batch, queue, r? \\ false) |
||||||
|
defp push_batch_to_queue([], queue, _r?), do: queue |
||||||
|
defp push_batch_to_queue(batch, queue, false), do: :queue.in(batch, queue) |
||||||
|
defp push_batch_to_queue(batch, queue, true), do: :queue.in_r(batch, queue) |
||||||
|
|
||||||
|
defp first_block do |
||||||
|
string_value = Application.get_env(:indexer, :first_block) |
||||||
|
|
||||||
|
case Integer.parse(string_value) do |
||||||
|
{integer, ""} -> |
||||||
|
integer |
||||||
|
|
||||||
|
_ -> |
||||||
|
min_missing_block_number = |
||||||
|
"min_missing_block_number" |
||||||
|
|> Chain.get_last_fetched_counter() |
||||||
|
|> Decimal.to_integer() |
||||||
|
|
||||||
|
min_missing_block_number |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
defp continue_future_updating?(max_fetched_block_number) do |
||||||
|
case Integer.parse(Application.get_env(:indexer, :last_block)) do |
||||||
|
{last_block, ""} -> max_fetched_block_number < last_block |
||||||
|
_ -> true |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
defp missing_ranges_batch_size do |
||||||
|
Application.get_env(:indexer, :missing_ranges_batch_size) || @default_missing_ranges_batch_size |
||||||
|
end |
||||||
|
|
||||||
|
def parse_block_ranges(block_ranges_string) do |
||||||
|
ranges = |
||||||
|
block_ranges_string |
||||||
|
|> String.split(",") |
||||||
|
|> Enum.map(fn string_range -> |
||||||
|
case String.split(string_range, "..") do |
||||||
|
[from_string, "latest"] -> |
||||||
|
parse_integer(from_string) |
||||||
|
|
||||||
|
[from_string, to_string] -> |
||||||
|
with {from, ""} <- Integer.parse(from_string), |
||||||
|
{to, ""} <- Integer.parse(to_string) do |
||||||
|
if from <= to, do: from..to, else: nil |
||||||
|
else |
||||||
|
_ -> nil |
||||||
|
end |
||||||
|
|
||||||
|
_ -> |
||||||
|
nil |
||||||
|
end |
||||||
|
end) |
||||||
|
|> Helper.sanitize_ranges() |
||||||
|
|
||||||
|
case List.last(ranges) do |
||||||
|
_from.._to -> |
||||||
|
{:finite_ranges, ranges} |
||||||
|
|
||||||
|
nil -> |
||||||
|
:no_ranges |
||||||
|
|
||||||
|
num -> |
||||||
|
{:infinite_ranges, List.delete_at(ranges, -1), num - 1} |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
defp parse_integer(integer_string) do |
||||||
|
case Integer.parse(integer_string) do |
||||||
|
{integer, ""} -> integer |
||||||
|
_ -> nil |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
@ -0,0 +1,83 @@ |
|||||||
|
defmodule Indexer.Block.Catchup.MissingRangesCollectorTest do |
||||||
|
use Explorer.DataCase |
||||||
|
|
||||||
|
alias Indexer.Block.Catchup.MissingRangesCollector |
||||||
|
|
||||||
|
test "default_init" do |
||||||
|
insert(:block, number: 1_000_000) |
||||||
|
insert(:block, number: 500_123) |
||||||
|
MissingRangesCollector.start_link([]) |
||||||
|
Process.sleep(500) |
||||||
|
|
||||||
|
assert [999_999..900_000//-1] = MissingRangesCollector.get_latest_batch() |
||||||
|
assert [899_999..800_000//-1] = MissingRangesCollector.get_latest_batch() |
||||||
|
assert [799_999..700_000//-1] = MissingRangesCollector.get_latest_batch() |
||||||
|
|
||||||
|
insert(:block, number: 1_200_000) |
||||||
|
Process.sleep(500) |
||||||
|
|
||||||
|
assert [1_199_999..1_100_001//-1] = MissingRangesCollector.get_latest_batch() |
||||||
|
assert [1_100_000..1_000_001//-1] = MissingRangesCollector.get_latest_batch() |
||||||
|
assert [699_999..600_000//-1] = MissingRangesCollector.get_latest_batch() |
||||||
|
assert [599_999..500_124//-1, 500_122..500_000//-1] = MissingRangesCollector.get_latest_batch() |
||||||
|
end |
||||||
|
|
||||||
|
describe "ranges_init" do |
||||||
|
setup do |
||||||
|
initial_env = Application.get_all_env(:indexer) |
||||||
|
on_exit(fn -> Application.put_all_env([{:indexer, initial_env}]) end) |
||||||
|
end |
||||||
|
|
||||||
|
test "infinite range" do |
||||||
|
Application.put_env(:indexer, :block_ranges, "1..5,3..5,2qw1..12,10..11a,,asd..qwe,10..latest") |
||||||
|
|
||||||
|
insert(:block, number: 200_000) |
||||||
|
|
||||||
|
MissingRangesCollector.start_link([]) |
||||||
|
Process.sleep(500) |
||||||
|
|
||||||
|
assert [199_999..100_010//-1] = MissingRangesCollector.get_latest_batch() |
||||||
|
assert [100_009..10//-1] = MissingRangesCollector.get_latest_batch() |
||||||
|
assert [5..1//-1] = MissingRangesCollector.get_latest_batch() |
||||||
|
end |
||||||
|
|
||||||
|
test "finite range" do |
||||||
|
Application.put_env(:indexer, :block_ranges, "10..20,5..15,18..25,35..40,30..50,150..200") |
||||||
|
|
||||||
|
insert(:block, number: 200_000) |
||||||
|
|
||||||
|
MissingRangesCollector.start_link([]) |
||||||
|
Process.sleep(500) |
||||||
|
|
||||||
|
assert [200..150//-1, 50..30//-1, 25..5//-1] = MissingRangesCollector.get_latest_batch() |
||||||
|
assert [] = MissingRangesCollector.get_latest_batch() |
||||||
|
end |
||||||
|
|
||||||
|
test "finite range with existing blocks" do |
||||||
|
Application.put_env(:indexer, :block_ranges, "10..20,5..15,18..25,35..40,30..50,150..200") |
||||||
|
|
||||||
|
insert(:block, number: 200_000) |
||||||
|
insert(:block, number: 175) |
||||||
|
insert(:block, number: 33) |
||||||
|
|
||||||
|
MissingRangesCollector.start_link([]) |
||||||
|
Process.sleep(500) |
||||||
|
|
||||||
|
assert [200..176//-1, 174..150//-1, 50..34//-1, 32..30//-1, 25..5//-1] = MissingRangesCollector.get_latest_batch() |
||||||
|
assert [] = MissingRangesCollector.get_latest_batch() |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
test "parse_block_ranges/1" do |
||||||
|
assert MissingRangesCollector.parse_block_ranges("1..5,3..5,2qw1..12,10..11a,,asd..qwe,10..latest") == |
||||||
|
{:infinite_ranges, [1..5], 9} |
||||||
|
|
||||||
|
assert MissingRangesCollector.parse_block_ranges("latest..123,,fvdskvjglav!@#$%^&,2..1") == :no_ranges |
||||||
|
|
||||||
|
assert MissingRangesCollector.parse_block_ranges("10..20,5..15,18..25,35..40,30..50,100..latest,150..200") == |
||||||
|
{:infinite_ranges, [5..25, 30..50], 99} |
||||||
|
|
||||||
|
assert MissingRangesCollector.parse_block_ranges("10..20,5..15,18..25,35..40,30..50,150..200") == |
||||||
|
{:finite_ranges, [5..25, 30..50, 150..200]} |
||||||
|
end |
||||||
|
end |
||||||
Loading…
Reference in new issue