Speedup intial query for catchup fetcher

.github/workflows/config.yml

@@ -19,7 +19,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-elixir@v1
         with:
-          otp-version: '23.2.7'
+          otp-version: '23.3.4.1'
           elixir-version: '1.11.3'
       - run: mix local.hex --force
       - run: mix local.rebar --force
@@ -46,7 +46,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-elixir@v1
         with:
-          otp-version: '23.2.7'
+          otp-version: '23.3.4.1'
           elixir-version: '1.11.3'
       - run: mix local.hex --force
       - run: mix local.rebar --force
@@ -61,7 +61,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-elixir@v1
         with:
-          otp-version: '23.2.7'
+          otp-version: '23.3.4.1'
           elixir-version: '1.11.3'
       - run: mix local.hex --force
       - run: mix local.rebar --force
@@ -76,7 +76,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-elixir@v1
         with:
-          otp-version: '23.2.7'
+          otp-version: '23.3.4.1'
           elixir-version: '1.11.3'
       - run: mix local.hex --force
       - run: mix local.rebar --force
@@ -103,7 +103,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-elixir@v1
         with:
-          otp-version: '23.2.7'
+          otp-version: '23.3.4.1'
           elixir-version: '1.11.3'
       - run: mix local.hex --force
       - run: mix local.rebar --force
@@ -121,7 +121,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-elixir@v1
         with:
-          otp-version: '23.2.7'
+          otp-version: '23.3.4.1'
           elixir-version: '1.11.3'
       - run: mix local.hex --force
       - run: mix local.rebar --force
@@ -142,7 +142,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-elixir@v1
         with:
-          otp-version: '23.2.7'
+          otp-version: '23.3.4.1'
           elixir-version: '1.11.3'
       - run: mix local.hex --force
       - run: mix local.rebar --force
@@ -167,7 +167,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-elixir@v1
         with:
-          otp-version: '23.2.7'
+          otp-version: '23.3.4.1'
           elixir-version: '1.11.3'
       - run: mix local.hex --force
       - run: mix local.rebar --force
@@ -211,7 +211,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-elixir@v1
         with:
-          otp-version: '23.2.7'
+          otp-version: '23.3.4.1'
           elixir-version: '1.11.3'
       - run: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
       - run: echo 'export PATH=~/.cargo/bin/:$PATH' >> $GITHUB_ENV
@@ -267,7 +267,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-elixir@v1
         with:
-          otp-version: '23.2.7'
+          otp-version: '23.3.4.1'
           elixir-version: '1.11.3'
       - run: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
       - run: echo 'export PATH=~/.cargo/bin/:$PATH' >> $GITHUB_ENV
@@ -325,7 +325,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-elixir@v1
         with:
-          otp-version: '23.2.7'
+          otp-version: '23.3.4.1'
           elixir-version: '1.11.3'
       - run: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
       - run: echo 'export PATH=~/.cargo/bin/:$PATH' >> $GITHUB_ENV
@@ -383,7 +383,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-elixir@v1
         with:
-          otp-version: '23.2.7'
+          otp-version: '23.3.4.1'
           elixir-version: '1.11.3'
       - run: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
       - run: echo 'export PATH=~/.cargo/bin/:$PATH' >> $GITHUB_ENV

CHANGELOG.md

@@ -11,6 +11,7 @@
 - [#4167](https://github.com/blockscout/blockscout/pull/4167) - Deduplicate block numbers in acquire_blocks function
 - [#4149](https://github.com/blockscout/blockscout/pull/4149) - Exclude smart_contract_additional_sources from JSON encoding in address schema
 - [#4137](https://github.com/blockscout/blockscout/pull/4137) - Get token balance query improvement
+- [#4129](https://github.com/blockscout/blockscout/pull/4129) - Speedup procedure of finding missing block numbers for catchup fetcher
 - [#4038](https://github.com/blockscout/blockscout/pull/4038) - Add clause for abi_decode_address_output/1 when is_nil(address)
 - [#3989](https://github.com/blockscout/blockscout/pull/3989), [4061](https://github.com/blockscout/blockscout/pull/4061) - Fixed bug that sometimes lead to incorrect ordering of token transfers
 - [#3946](https://github.com/blockscout/blockscout/pull/3946) - Get NFT metadata from URIs with status_code 301

apps/explorer/lib/explorer/application.ex

@@ -15,6 +15,7 @@ defmodule Explorer.Application do
     BlockNumber,
     Blocks,
     GasUsage,
+    MinMissingBlockNumber,
     NetVersion,
     TransactionCount,
     Transactions,
@@ -60,7 +61,8 @@ defmodule Explorer.Application do
       con_cache_child_spec(RSK.cache_name(), ttl_check_interval: :timer.minutes(1), global_ttl: :timer.minutes(30)),
       Transactions,
       Accounts,
-      Uncles
+      Uncles,
+      MinMissingBlockNumber
     ]
 
     children = base_children ++ configurable_children()

apps/explorer/lib/explorer/chain.ex

@@ -1604,6 +1604,9 @@ defmodule Explorer.Chain do
       )
 
     Repo.one(query) || 0
+  rescue
+    _ ->
+      0
   end
 
   @spec fetch_max_block_number() :: non_neg_integer
@@ -1617,6 +1620,9 @@ defmodule Explorer.Chain do
       )
 
     Repo.one(query) || 0
+  rescue
+    _ ->
+      0
   end
 
   @spec fetch_count_consensus_block() :: non_neg_integer
@@ -2473,7 +2479,7 @@ defmodule Explorer.Chain do
         select: last_fetched_counter.value
       )
 
-    Repo.one!(query) || 0
+    Repo.one!(query) || Decimal.new(0)
   end
 
   defp block_status({number, timestamp}) do
@@ -2489,6 +2495,33 @@ defmodule Explorer.Chain do
 
   defp block_status(nil), do: {:error, :no_blocks}
 
+  def fetch_min_missing_block_cache do
+    max_block_number = BlockNumber.get_max()
+
+    if max_block_number > 0 do
+      query =
+        from(b in Block,
+          right_join:
+            missing_range in fragment(
+              """
+                (SELECT b1.number 
+                FROM generate_series(0, (?)::integer) AS b1(number)
+                WHERE NOT EXISTS
+                  (SELECT 1 FROM blocks b2 WHERE b2.number=b1.number AND b2.consensus))
+              """,
+              ^max_block_number
+            ),
+          on: b.number == missing_range.number,
+          select: min(missing_range.number)
+        )
+
+      query
+      |> Repo.one() || 0
+    else
+      0
+    end
+  end
+
   @doc """
   Calculates the ranges of missing consensus blocks in `range`.
 

apps/explorer/lib/explorer/chain/cache/block_number.ex

@@ -34,7 +34,5 @@ defmodule Explorer.Chain.Cache.BlockNumber do
       :min -> Chain.fetch_min_block_number()
       :max -> Chain.fetch_max_block_number()
     end
-  rescue
-    _e -> 0
   end
 end

apps/explorer/lib/explorer/chain/cache/min_missing_block.ex

@@ -0,0 +1,52 @@
+defmodule Explorer.Chain.Cache.MinMissingBlockNumber do
+  @moduledoc """
+  Caches min missing block number (break in the chain).
+  """
+
+  use GenServer
+
+  alias Explorer.Chain
+
+  @doc """
+  Starts a process to periodically update the % of blocks indexed.
+  """
+  @spec start_link(term()) :: GenServer.on_start()
+  def start_link(_) do
+    GenServer.start_link(__MODULE__, :ok, name: __MODULE__)
+  end
+
+  @impl true
+  def init(args) do
+    Task.start_link(&fetch_min_missing_block/0)
+
+    schedule_next_consolidation()
+
+    {:ok, args}
+  end
+
+  def fetch_min_missing_block do
+    result = Chain.fetch_min_missing_block_cache()
+
+    if result > 0 do
+      params = %{
+        counter_type: "min_missing_block_number",
+        value: result
+      }
+
+      Chain.upsert_last_fetched_counter(params)
+    end
+  end
+
+  defp schedule_next_consolidation do
+    Process.send_after(self(), :fetch_min_missing_block, :timer.minutes(20))
+  end
+
+  @impl true
+  def handle_info(:fetch_min_missing_block, state) do
+    fetch_min_missing_block()
+
+    schedule_next_consolidation()
+
+    {:noreply, state}
+  end
+end

apps/explorer/priv/repo/migrations/20210524165427_min_missing_block_number.exs

@@ -0,0 +1,12 @@
+defmodule Explorer.Repo.Migrations.MinMissingBlockNumber do
+  use Ecto.Migration
+
+  def change do
+    insert_initial_genesis_block = """
+    INSERT INTO last_fetched_counters (counter_type, value, inserted_at, updated_at)
+      VALUES ('min_missing_block_number', 0, NOW(), NOW());
+    """
+
+    execute(insert_initial_genesis_block)
+  end
+end

apps/indexer/config/config.exs

@@ -35,7 +35,7 @@ config :indexer,
     String.to_integer(System.get_env("TOKEN_METADATA_UPDATE_INTERVAL") || "#{2 * 24 * 60 * 60}"),
   # bytes
   memory_limit: 1 <<< 30,
-  first_block: System.get_env("FIRST_BLOCK") || "0",
+  first_block: System.get_env("FIRST_BLOCK") || "",
   last_block: System.get_env("LAST_BLOCK") || ""
 
 config :indexer, Indexer.Fetcher.PendingTransaction.Supervisor,

apps/indexer/lib/indexer/block/catchup/fetcher.ex

@@ -340,8 +340,16 @@ defmodule Indexer.Block.Catchup.Fetcher do
     string_value = Application.get_env(:indexer, :first_block)
 
     case Integer.parse(string_value) do
-      {integer, ""} -> integer
+      {integer, ""} ->
-      _ -> 0
+        integer
+
+      _ ->
+        min_missing_block_number =
+          "min_missing_block_number"
+          |> Chain.get_last_fetched_counter()
+          |> Decimal.to_integer()
+
+        min_missing_block_number
     end
   end