Merge pull request #7393 from blockscout/realtime-fetcher-max-gap

Realtime fetcher max gap

CHANGELOG.md

@@ -5,6 +5,7 @@
 ### Features
 
 - [#7355](https://github.com/blockscout/blockscout/pull/7355) - Add endpoint for token info import
+- [#7393](https://github.com/blockscout/blockscout/pull/7393) - Realtime fetcher max gap
 
 ### Fixes
 - [#7391](https://github.com/blockscout/blockscout/pull/7391) - Fix: cannot read properties of null (reading 'value')

apps/indexer/lib/indexer/block/realtime/fetcher.ex

@@ -124,9 +124,14 @@ defmodule Indexer.Block.Realtime.Fetcher do
     new_previous_number =
       case EthereumJSONRPC.fetch_block_number_by_tag("latest", json_rpc_named_arguments) do
         {:ok, number} when is_nil(previous_number) or number != previous_number ->
-          start_fetch_and_import(number, block_fetcher, previous_number)
-
-          number
+          if abnormal_gap?(number, previous_number) do
+            new_number = max(number, previous_number)
+            start_fetch_and_import(new_number, block_fetcher, previous_number)
+            new_number
+          else
+            start_fetch_and_import(number, block_fetcher, previous_number)
+            number
+          end
 
         _ ->
           previous_number
@@ -263,6 +268,15 @@ defmodule Indexer.Block.Realtime.Fetcher do
 
   defp reorg?(_, _), do: false
 
+  @default_max_gap 1000
+  defp abnormal_gap?(_number, nil), do: false
+
+  defp abnormal_gap?(number, previous_number) do
+    max_gap = Application.get_env(:indexer, __MODULE__)[:max_gap] || @default_max_gap
+
+    abs(number - previous_number) > max_gap
+  end
+
   @reorg_delay 5_000
 
   @decorate trace(name: "fetch", resource: "Indexer.Block.Realtime.Fetcher.fetch_and_import_block/3", tracer: Tracer)

config/runtime.exs

@@ -463,6 +463,9 @@ config :indexer, Indexer.Fetcher.TokenInstance.Sanitize.Supervisor,
 config :indexer, Indexer.Fetcher.EmptyBlocksSanitizer,
   batch_size: ConfigHelper.parse_integer_env_var("INDEXER_EMPTY_BLOCKS_SANITIZER_BATCH_SIZE", 100)
 
+config :indexer, Indexer.Block.Realtime.Fetcher,
+  max_gap: ConfigHelper.parse_integer_env_var("INDEXER_REALTIME_FETCHER_MAX_GAP", 1000)
+
 config :indexer, Indexer.Block.Catchup.MissingRangesCollector,
   batch_size: ConfigHelper.parse_integer_env_var("INDEXER_CATCHUP_MISSING_RANGES_BATCH_SIZE", 100_000)
 

docker-compose/envs/common-blockscout.env

@@ -116,6 +116,7 @@ INDEXER_DISABLE_INTERNAL_TRANSACTIONS_FETCHER=false
 # INDEXER_TX_ACTIONS_REINDEX_LAST_BLOCK=
 # INDEXER_TX_ACTIONS_REINDEX_PROTOCOLS=
 # INDEXER_TX_ACTIONS_AAVE_V3_POOL_CONTRACT=
+# INDEXER_REALTIME_FETCHER_MAX_GAP=
 # TOKEN_ID_MIGRATION_FIRST_BLOCK=
 # TOKEN_ID_MIGRATION_CONCURRENCY=
 # TOKEN_ID_MIGRATION_BATCH_SIZE=

docker/Makefile

@@ -549,6 +549,9 @@ endif
 ifdef INDEXER_EMPTY_BLOCKS_SANITIZER_BATCH_SIZE
     BLOCKSCOUT_CONTAINER_PARAMS += -e 'INDEXER_EMPTY_BLOCKS_SANITIZER_BATCH_SIZE=$(INDEXER_EMPTY_BLOCKS_SANITIZER_BATCH_SIZE)'
 endif
+ifdef INDEXER_REALTIME_FETCHER_MAX_GAP
+    BLOCKSCOUT_CONTAINER_PARAMS += -e 'INDEXER_REALTIME_FETCHER_MAX_GAP=$(INDEXER_REALTIME_FETCHER_MAX_GAP)'
+endif
 ifdef INDEXER_INTERNAL_TRANSACTIONS_TRACER_TYPE
     BLOCKSCOUT_CONTAINER_PARAMS += -e 'INDEXER_INTERNAL_TRANSACTIONS_TRACER_TYPE=$(INDEXER_INTERNAL_TRANSACTIONS_TRACER_TYPE)'
 endif