From f0ef80be8d7accf96f6f5d3eee8668582516d3b8 Mon Sep 17 00:00:00 2001 From: Alexander Kolotov Date: Fri, 20 Sep 2024 02:47:14 -0600 Subject: [PATCH] fix: get rid of heavy DB query to start Arbitrum missed messages discovery process (#10767) * simplified approach to start discovery * function comment improved --- .../arbitrum/rollup_messages_catchup.ex | 51 +++++------ .../lib/indexer/fetcher/arbitrum/utils/db.ex | 89 +------------------ 2 files changed, 27 insertions(+), 113 deletions(-) diff --git a/apps/indexer/lib/indexer/fetcher/arbitrum/rollup_messages_catchup.ex b/apps/indexer/lib/indexer/fetcher/arbitrum/rollup_messages_catchup.ex index a0ec6e0c8a..56db328edd 100644 --- a/apps/indexer/lib/indexer/fetcher/arbitrum/rollup_messages_catchup.ex +++ b/apps/indexer/lib/indexer/fetcher/arbitrum/rollup_messages_catchup.ex @@ -64,7 +64,7 @@ defmodule Indexer.Fetcher.Arbitrum.RollupMessagesCatchup do require Logger @wait_for_new_block_delay 15 - @release_cpu_delay 1 + @release_cpu_delay 2 def child_spec(start_link_arguments) do spec = %{ @@ -167,43 +167,44 @@ defmodule Indexer.Fetcher.Arbitrum.RollupMessagesCatchup do {:noreply, %{state | data: new_data}} end - # Sets the initial parameters for discovering historical messages. This function - # inspects the database for missed messages and, if any are found, identifies the - # end blocks of the ranges for both L1-to-L2 and L2-to-L1 messages. If no missed - # messages are found, the block number before the latest indexed block will be used. - # These end blocks are used to initiate the discovery process in subsequent iterations. + # Sets the end blocks of the ranges for discovering historical L1-to-L2 and L2-to-L1 messages. # - # After identifying the initial values, the function immediately transitions to - # the L2-to-L1 message discovery process by sending the `:historical_msg_from_l2` - # message. + # There is likely a way to query the DB and discover the exact block of the + # first missed message (both L1-to-L2 and L2-to-L1) and start the discovery + # process from there. However, such a query is very expensive and can take a + # long time for chains with a high number of transactions. Instead, it's + # possible to start looking for missed messages from the block before the + # latest indexed block. + # + # Although this approach is not optimal for Blockscout instances where there + # are no missed messages (assumed to be the majority), it is still preferable + # to the first approach. The reason is that a finite number of relatively + # cheap queries (which can be tuned with `missed_messages_blocks_depth`) are + # preferable to one expensive query that becomes even more expensive as the + # number of indexed transactions grows. + # + # After identifying the initial values, the function immediately transitions + # to the L2-to-L1 message discovery process by sending the + # `:historical_msg_from_l2` message. # # ## Parameters # - `:init_worker`: The message that triggers the handler. - # - `state`: The current state of the fetcher containing the first rollup block - # number and the number of the most recent block indexed. + # - `state`: The current state of the fetcher containing the number of the + # most recent block indexed. # # ## Returns - # - `{:noreply, new_state}` where the end blocks for both L1-to-L2 and L2-to-L1 - # message discovery are established. + # - `{:noreply, new_state}` where `new_state` contains the updated state with + # end blocks for both L1-to-L2 and L2-to-L1 message discovery established. @impl GenServer - def handle_info( - :init_worker, - %{config: %{rollup_rpc: %{first_block: rollup_first_block}}, data: %{new_block: just_received_block}} = state - ) do - historical_msg_from_l2_end_block = - Db.rollup_block_to_discover_missed_messages_from_l2(just_received_block, rollup_first_block) - - historical_msg_to_l2_end_block = - Db.rollup_block_to_discover_missed_messages_to_l2(just_received_block, rollup_first_block) - + def handle_info(:init_worker, %{data: %{new_block: just_received_block}} = state) do Process.send(self(), :historical_msg_from_l2, []) new_data = Map.merge(state.data, %{ duration: 0, progressed: false, - historical_msg_from_l2_end_block: historical_msg_from_l2_end_block, - historical_msg_to_l2_end_block: historical_msg_to_l2_end_block + historical_msg_from_l2_end_block: just_received_block, + historical_msg_to_l2_end_block: just_received_block }) {:noreply, %{state | data: new_data}} diff --git a/apps/indexer/lib/indexer/fetcher/arbitrum/utils/db.ex b/apps/indexer/lib/indexer/fetcher/arbitrum/utils/db.ex index 27a9b11bef..9f7b6c8778 100644 --- a/apps/indexer/lib/indexer/fetcher/arbitrum/utils/db.ex +++ b/apps/indexer/lib/indexer/fetcher/arbitrum/utils/db.ex @@ -3,7 +3,7 @@ defmodule Indexer.Fetcher.Arbitrum.Utils.Db do Common functions to simplify DB routines for Indexer.Fetcher.Arbitrum fetchers """ - import Indexer.Fetcher.Arbitrum.Utils.Logging, only: [log_warning: 1, log_info: 1] + import Indexer.Fetcher.Arbitrum.Utils.Logging, only: [log_warning: 1] alias Explorer.Chain alias Explorer.Chain.Arbitrum @@ -224,83 +224,6 @@ defmodule Indexer.Fetcher.Arbitrum.Utils.Db do end end - @doc """ - Determines the rollup block number to discover missed L2-to-L1 messages within - a specified range. - - The function checks for the first missed L2-to-L1 message and whether historical - block fetching is still in progress. If no missed messages are found and - historical fetching is complete, it returns the block number just before the - first rollup block. Otherwise, it returns the appropriate block number based on - the findings. - - ## Parameters - - `initial_value`: The initial block number to start the further search of the - missed messages from if no missed messages are found and historical blocks - are not fetched yet. - - `rollup_first_block`: The block number of the first rollup block. - - ## Returns - - The block number of the first missed L2-to-L1 message. - """ - @spec rollup_block_to_discover_missed_messages_from_l2(FullBlock.block_number(), FullBlock.block_number()) :: - nil | FullBlock.block_number() - def rollup_block_to_discover_missed_messages_from_l2(initial_value, rollup_first_block) do - arbsys_contract = Application.get_env(:indexer, Indexer.Fetcher.Arbitrum.Messaging)[:arbsys_contract] - - with {:block, nil} <- - {:block, Reader.rollup_block_of_first_missed_message_from_l2(arbsys_contract, @l2_to_l1_event)}, - {:synced, true} <- {:synced, rollup_synced?()} do - log_info("No missed messages from L2 found") - rollup_first_block - 1 - else - {:block, value} -> - log_info("First missed message from L2 found in block #{value}") - value - - {:synced, false} -> - log_info("No missed messages from L2 found but historical blocks fetching still in progress") - initial_value - end - end - - @doc """ - Determines the rollup block number to discover missed L1-to-L2 messages within - a specified range. - - The function checks for the first missed L1-to-L2 message and whether historical - block fetching is still in progress. If no missed messages are found and - historical fetching is complete, it returns the block number just before the - first rollup block. Otherwise, it returns the appropriate block number based on - the findings. - - ## Parameters - - `initial_value`: The initial block number to start the further search of the - missed messages from if no missed messages are found and historical blocks - are not fetched yet. - - `rollup_first_block`: The block number of the first rollup block. - - ## Returns - - The block number of the first missed L1-to-L2 message. - """ - @spec rollup_block_to_discover_missed_messages_to_l2(FullBlock.block_number(), FullBlock.block_number()) :: - nil | FullBlock.block_number() - def rollup_block_to_discover_missed_messages_to_l2(initial_value, rollup_first_block) do - with {:block, nil} <- {:block, Reader.rollup_block_of_first_missed_message_to_l2()}, - {:synced, true} <- {:synced, rollup_synced?()} do - log_info("No missed messages to L2 found") - rollup_first_block - 1 - else - {:block, value} -> - log_info("First missed message to L2 found in block #{value}") - value - - {:synced, false} -> - log_info("No missed messages to L2 found but historical blocks fetching still in progress") - initial_value - end - end - @doc """ Retrieves the L1 block number immediately following the block where the confirmation transaction for the highest confirmed rollup block was included. @@ -969,16 +892,6 @@ defmodule Indexer.Fetcher.Arbitrum.Utils.Db do Reader.get_da_info_by_batch_number(batch_number) end - # Checks if the rollup is synced by verifying if the block after the first block exists in the database. - @spec rollup_synced?() :: boolean() - defp rollup_synced? do - # Since zero block does not have any useful data, it make sense to consider - # the block just after it - rollup_tail = Application.get_all_env(:indexer)[:first_block] + 1 - - Reader.rollup_block_exists?(rollup_tail) - end - @spec lifecycle_transaction_to_map(Arbitrum.LifecycleTransaction.t()) :: Arbitrum.LifecycleTransaction.to_import() defp lifecycle_transaction_to_map(tx) do [:id, :hash, :block_number, :timestamp, :status]