Find min_missing_block_number by batches

apps/explorer/lib/explorer/chain.ex

@@ -3029,21 +3029,23 @@ defmodule Explorer.Chain do
 
   defp block_status(nil), do: {:error, :no_blocks}
 
-  def fetch_min_missing_block_cache do
+  def fetch_min_missing_block_cache(from \\ nil, to \\ nil) do
-    max_block_number = BlockNumber.get_max()
+    from_block_number = from || 0
+    to_block_number = to || BlockNumber.get_max()
 
-    if max_block_number > 0 do
+    if to_block_number > 0 do
       query =
         from(b in Block,
           right_join:
             missing_range in fragment(
               """
                 (SELECT b1.number
-                FROM generate_series(0, (?)::integer) AS b1(number)
+                FROM generate_series((?)::integer, (?)::integer) AS b1(number)
                 WHERE NOT EXISTS
                   (SELECT 1 FROM blocks b2 WHERE b2.number=b1.number AND b2.consensus))
               """,
-              ^max_block_number
+              ^from_block_number,
+              ^to_block_number
             ),
           on: b.number == missing_range.number,
           select: min(missing_range.number)

apps/explorer/lib/explorer/chain/cache/min_missing_block.ex

@@ -6,6 +6,12 @@ defmodule Explorer.Chain.Cache.MinMissingBlockNumber do
   use GenServer
 
   alias Explorer.Chain
+  alias Explorer.Chain.Cache.BlockNumber
+
+  @default_batch_size 100_000
+  @normal_interval 10
+  @increased_interval :timer.minutes(20)
+  @default_last_fetched_number -1
 
   @doc """
   Starts a process to periodically update the % of blocks indexed.
@@ -16,37 +22,52 @@ defmodule Explorer.Chain.Cache.MinMissingBlockNumber do
   end
 
   @impl true
-  def init(args) do
+  def init(_) do
-    Task.start_link(&fetch_min_missing_block/0)
+    schedule_next_consolidation(@normal_interval)
+    {:ok, %{last_fetched_number: @default_last_fetched_number}}
+  end
 
-    schedule_next_consolidation()
+  def fetch_min_missing_block(last_fetched_number) do
+    from = last_fetched_number + 1
+    to = last_fetched_number + batch_size()
+    max_block_number = BlockNumber.get_max() - 1
 
-    {:ok, args}
+    {corrected_to, continue?} = if to >= max_block_number, do: {max_block_number, false}, else: {to, true}
-  end
+
+    result = Chain.fetch_min_missing_block_cache(from, corrected_to)
 
-  def fetch_min_missing_block do
+    cond do
-    result = Chain.fetch_min_missing_block_cache()
+      not is_nil(result) ->
+        params = %{
+          counter_type: "min_missing_block_number",
+          value: result
+        }
 
-    unless is_nil(result) do
+        Chain.upsert_last_fetched_counter(params)
-      params = %{
+        schedule_next_consolidation(@increased_interval)
-        counter_type: "min_missing_block_number",
+        @default_last_fetched_number
-        value: result
-      }
 
-      Chain.upsert_last_fetched_counter(params)
+      continue? ->
+        schedule_next_consolidation(@normal_interval)
+        corrected_to
+
+      true ->
+        schedule_next_consolidation(@increased_interval)
+        @default_last_fetched_number
     end
   end
 
-  defp schedule_next_consolidation do
+  defp schedule_next_consolidation(interval) do
-    Process.send_after(self(), :fetch_min_missing_block, :timer.minutes(20))
+    Process.send_after(self(), :fetch_min_missing_block, interval)
   end
 
   @impl true
-  def handle_info(:fetch_min_missing_block, state) do
+  def handle_info(:fetch_min_missing_block, %{last_fetched_number: last_fetched_number} = state) do
-    fetch_min_missing_block()
+    new_last_number = fetch_min_missing_block(last_fetched_number)
-
+    {:noreply, %{state | last_fetched_number: new_last_number}}
-    schedule_next_consolidation()
+  end
 
-    {:noreply, state}
+  defp batch_size do
+    Application.get_env(:explorer, __MODULE__)[:batch_size] || @default_batch_size
   end
 end

config/runtime.exs

@@ -378,6 +378,13 @@ config :explorer, :token_id_migration,
   concurrency: token_id_migration_concurrency,
   batch_size: token_id_migration_batch_size
 
+min_missing_block_number_batch_size_default_str = "100000"
+
+{min_missing_block_number_batch_size, _} =
+  Integer.parse(System.get_env("MIN_MISSING_BLOCK_NUMBER_BATCH_SIZE", min_missing_block_number_batch_size_default_str))
+
+config :explorer, Explorer.Chain.Cache.MinMissingBlockNumber, batch_size: min_missing_block_number_batch_size
+
 ###############
 ### Indexer ###
 ###############