Merge pull request #6407 from blockscout/vb-calc-int-txs-indexing-ratio

Indexed ratio for int txs fetching stage

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ### Features
 
+- [#6407](https://github.com/blockscout/blockscout/pull/6407) - Indexed ratio for int txs fetching stage
 - [#6324](https://github.com/blockscout/blockscout/pull/6324) - Add verified contracts list page
 - [#6379](https://github.com/blockscout/blockscout/pull/6379) - API v2 for frontend
 - [#6351](https://github.com/blockscout/blockscout/pull/6351) - Enable forum link env var

apps/block_scout_web/assets/js/lib/indexing.js

@@ -3,23 +3,48 @@ import humps from 'humps'
 import numeral from 'numeral'
 import socket from '../socket'
 
-function tryUpdateIndexedStatus (el, indexedRatio = el.dataset.indexedRatio, indexingFinished = false) {
+function tryUpdateIndexedStatus (el, indexedRatioBlocks = el.dataset.indexedRatioBlocks, indexedRatio = el.dataset.indexedRatio, indexingFinished = false) {
   if (indexingFinished) return $("[data-selector='indexed-status']").remove()
-  const blocksPercentComplete = numeral(indexedRatio).format('0%')
+  const indexedRatioFloat = parseFloat(indexedRatio)
+  const indexedRatioBlocksFloat = parseFloat(indexedRatioBlocks)
+
+  if (!isNaN(indexedRatioBlocksFloat)) {
+    el.dataset.indexedRatioBlocks = indexedRatioBlocks
+  } else if (!isNaN(indexedRatioFloat)) {
+    el.dataset.indexedRatio = indexedRatio
+  }
+
+  const blocksPercentComplete = numeral(el.dataset.indexedRatioBlocks).format('0%')
   let indexedText
   if (blocksPercentComplete === '100%') {
-    indexedText = window.localized['Indexing Internal Transactions']
+    const intTxsPercentComplete = numeral(el.dataset.indexedRatio).format('0%')
+    indexedText = `${intTxsPercentComplete} ${window.localized['Blocks With Internal Transactions Indexed']}`
   } else {
     indexedText = `${blocksPercentComplete} ${window.localized['Blocks Indexed']}`
   }
-  if (indexedText !== el.innerHTML) el.innerHTML = indexedText
+
+  if (indexedText !== el.innerHTML) {
+    el.innerHTML = indexedText
+  }
 }
 
-export function updateIndexStatus (msg = {}) {
-  $('[data-indexed-ratio]').each((i, el) => tryUpdateIndexedStatus(el, msg.ratio, msg.finished))
+export function updateIndexStatus (msg = {}, type) {
+  $('[data-indexed-ratio]').each((i, el) => {
+    if (type === 'blocks') {
+      tryUpdateIndexedStatus(el, msg.ratio, null, msg.finished)
+    } else if (type === 'internal_transactions') {
+      tryUpdateIndexedStatus(el, null, msg.ratio, msg.finished)
+    } else {
+      tryUpdateIndexedStatus(el, null, null, msg.finished)
+    }
+  })
 }
 updateIndexStatus()
 
-const indexingChannel = socket.channel('blocks:indexing')
-indexingChannel.join()
-indexingChannel.on('index_status', (msg) => updateIndexStatus(humps.camelizeKeys(msg)))
+const IndexingChannelBlocks = socket.channel('blocks:indexing')
+IndexingChannelBlocks.join()
+IndexingChannelBlocks.on('index_status', (msg) => updateIndexStatus(humps.camelizeKeys(msg), 'blocks'))
+
+const indexingChannelInternalTransactions = socket.channel('blocks:indexing_internal_transactions')
+indexingChannelInternalTransactions.join()
+indexingChannelInternalTransactions.on('index_status', (msg) => updateIndexStatus(humps.camelizeKeys(msg), 'internal_transactions'))

apps/block_scout_web/config/config.exs

@@ -33,6 +33,8 @@ config :block_scout_web,
 
 config :block_scout_web, BlockScoutWeb.Counters.BlocksIndexedCounter, enabled: true
 
+config :block_scout_web, BlockScoutWeb.Counters.InternalTransactionsIndexedCounter, enabled: true
+
 # Configures the endpoint
 config :block_scout_web, BlockScoutWeb.Endpoint,
   url: [

apps/block_scout_web/config/test.exs

@@ -22,6 +22,8 @@ config :wallaby, screenshot_on_failure: true, driver: Wallaby.Chrome, js_errors:
 
 config :block_scout_web, BlockScoutWeb.Counters.BlocksIndexedCounter, enabled: false
 
+config :block_scout_web, BlockScoutWeb.Counters.InternalTransactionsIndexedCounter, enabled: false
+
 config :block_scout_web, :captcha_helper, BlockScoutWeb.TestCaptchaHelper
 
 config :ueberauth, Ueberauth,

apps/block_scout_web/lib/block_scout_web/application.ex

@@ -6,7 +6,7 @@ defmodule BlockScoutWeb.Application do
   use Application
 
   alias BlockScoutWeb.API.APILogger
-  alias BlockScoutWeb.Counters.BlocksIndexedCounter
+  alias BlockScoutWeb.Counters.{BlocksIndexedCounter, InternalTransactionsIndexedCounter}
   alias BlockScoutWeb.{Endpoint, Prometheus}
   alias BlockScoutWeb.RealtimeEventHandler
 
@@ -35,7 +35,8 @@ defmodule BlockScoutWeb.Application do
       child_spec(Endpoint, []),
       {Absinthe.Subscription, Endpoint},
       {RealtimeEventHandler, name: RealtimeEventHandler},
-      {BlocksIndexedCounter, name: BlocksIndexedCounter}
+      {BlocksIndexedCounter, name: BlocksIndexedCounter},
+      {InternalTransactionsIndexedCounter, name: InternalTransactionsIndexedCounter}
     ]
 
     opts = [strategy: :one_for_one, name: BlockScoutWeb.Supervisor, max_restarts: 1_000]

apps/block_scout_web/lib/block_scout_web/counters/blocks_indexed_counter.ex

@@ -36,11 +36,11 @@ defmodule BlockScoutWeb.Counters.BlocksIndexedCounter do
   end
 
   def calculate_blocks_indexed do
-    indexed_ratio = Chain.indexed_ratio()
+    indexed_ratio_blocks = Chain.indexed_ratio_blocks()
 
-    finished? = Chain.finished_indexing?(indexed_ratio)
+    finished? = Chain.finished_indexing?(indexed_ratio_blocks)
 
-    Notifier.broadcast_blocks_indexed_ratio(indexed_ratio, finished?)
+    Notifier.broadcast_blocks_indexed_ratio(indexed_ratio_blocks, finished?)
   end
 
   defp schedule_next_consolidation do

apps/block_scout_web/lib/block_scout_web/counters/internal_transactions_indexed_counter.ex

@@ -0,0 +1,58 @@
+defmodule BlockScoutWeb.Counters.InternalTransactionsIndexedCounter do
+  @moduledoc """
+  Module responsible for fetching and consolidating the number pending block operations (internal transactions) indexed.
+
+  It loads the count asynchronously in a time interval.
+  """
+
+  use GenServer
+
+  alias BlockScoutWeb.Notifier
+  alias Explorer.Chain
+
+  # It is undesirable to automatically start the counter in all environments.
+  # Consider the test environment: if it initiates but does not finish before a
+  # test ends, that test will fail.
+  config = Application.compile_env(:block_scout_web, __MODULE__)
+  @enabled Keyword.get(config, :enabled)
+
+  @doc """
+  Starts a process to periodically update the % of internal transactions indexed.
+  """
+  @spec start_link(term()) :: GenServer.on_start()
+  def start_link(_) do
+    GenServer.start_link(__MODULE__, :ok, name: __MODULE__)
+  end
+
+  @impl true
+  def init(args) do
+    if @enabled do
+      Task.start_link(&calculate_internal_transactions_indexed/0)
+
+      schedule_next_consolidation()
+    end
+
+    {:ok, args}
+  end
+
+  def calculate_internal_transactions_indexed do
+    indexed_ratio_internal_transactions = Chain.indexed_ratio_internal_transactions()
+
+    finished? = Chain.finished_indexing?(indexed_ratio_internal_transactions)
+
+    Notifier.broadcast_internal_transactions_indexed_ratio(indexed_ratio_internal_transactions, finished?)
+  end
+
+  defp schedule_next_consolidation do
+    Process.send_after(self(), :calculate_internal_transactions_indexed, :timer.minutes(7))
+  end
+
+  @impl true
+  def handle_info(:calculate_internal_transactions_indexed, state) do
+    calculate_internal_transactions_indexed()
+
+    schedule_next_consolidation()
+
+    {:noreply, state}
+  end
+end

apps/block_scout_web/lib/block_scout_web/notifier.ex

@@ -242,6 +242,16 @@ defmodule BlockScoutWeb.Notifier do
     })
   end
 
+  @doc """
+  Broadcast the percentage of pending block operations indexed so far.
+  """
+  def broadcast_internal_transactions_indexed_ratio(ratio, finished?) do
+    Endpoint.broadcast("blocks:indexing_internal_transactions", "index_status", %{
+      ratio: Decimal.to_string(ratio),
+      finished: finished?
+    })
+  end
+
   defp broadcast_latest_block?(block, last_broadcasted_block_number) do
     cond do
       last_broadcasted_block_number == 0 || last_broadcasted_block_number == block.number - 1 ||

apps/block_scout_web/lib/block_scout_web/templates/layout/app.html.eex

@@ -41,7 +41,7 @@
       window.localized = {
         'Blocks Indexed': '<%= gettext("Blocks Indexed") %>',
         'Block Processing': '<%= gettext("Block Mined, awaiting import...") %>',
-        'Indexing Internal Transactions': '<%= gettext("Indexing Internal Transactions") %>',
+        'Blocks With Internal Transactions Indexed': '<%= gettext("Blocks With Internal Transactions Indexed") %>',
         'Less than': '<%= gettext("Less than") %>',
         'Market Cap': '<%= gettext("Market Cap") %>',
         'Price': '<%= gettext("Price") %>',
@@ -217,11 +217,16 @@
           <%= raw(System.get_env("MAINTENANCE_ALERT_MESSAGE")) %>
         </div>
       <% end %>
-      <% indexed_ratio = Explorer.Chain.indexed_ratio() %>
-      <%= if not Explorer.Chain.finished_indexing?(indexed_ratio) do %>
+      <% indexed_ratio_blocks = Explorer.Chain.indexed_ratio_blocks() %>
+      <% indexed_ratio =
+      case Chain.finished_blocks_indexing?(indexed_ratio_blocks) do
+       false -> indexed_ratio_blocks
+       _ -> Explorer.Chain.indexed_ratio_internal_transactions()
+      end %>
+      <%= if not Explorer.Chain.finished_indexing?(indexed_ratio_blocks) do %>
         <div class="alert alert-warning text-center mb-0 p-3" data-seindexed_ratiolector="indexed-status">
           <%= render BlockScoutWeb.CommonComponentsView, "_loading_spinner.html" %>
-          <span data-indexed-ratio="<%= indexed_ratio %>"></span>
+          <span data-indexed-ratio-blocks="<%= indexed_ratio_blocks %>" data-indexed-ratio="<%= indexed_ratio %>"></span>
           <%= gettext("- We're indexing this chain right now. Some of the counts may be inaccurate.") %>
         </div>
       <% end %>

apps/block_scout_web/priv/gettext/default.pot

@@ -76,7 +76,7 @@ msgstr ""
 msgid "(query)"
 msgstr ""
 
-#: lib/block_scout_web/templates/layout/app.html.eex:225
+#: lib/block_scout_web/templates/layout/app.html.eex:230
 #, elixir-autogen, elixir-format
 msgid "- We're indexing this chain right now. Some of the counts may be inaccurate."
 msgstr ""
@@ -3254,11 +3254,6 @@ msgstr ""
 msgid "truffle flattener"
 msgstr ""
 
-#: lib/block_scout_web/templates/layout/app.html.eex:44
-#, elixir-autogen, elixir-format
-msgid "Indexing Internal Transactions"
-msgstr ""
-
 #: lib/block_scout_web/templates/address_contract_verification_common_fields/_library_first.html.eex:5
 #, elixir-autogen, elixir-format
 msgid ") may be added for each contract. Click the Add Library button to add an additional one."
@@ -3424,3 +3419,8 @@ msgstr ""
 #, elixir-autogen, elixir-format
 msgid "Verifed contracts, %{subnetwork}, %{coin}"
 msgstr ""
+
+#: lib/block_scout_web/templates/layout/app.html.eex:44
+#, elixir-autogen, elixir-format
+msgid "Blocks With Internal Transactions Indexed"
+msgstr ""

apps/block_scout_web/priv/gettext/en/LC_MESSAGES/default.po

@@ -76,7 +76,7 @@ msgstr ""
 msgid "(query)"
 msgstr ""
 
-#: lib/block_scout_web/templates/layout/app.html.eex:225
+#: lib/block_scout_web/templates/layout/app.html.eex:230
 #, elixir-autogen, elixir-format
 msgid "- We're indexing this chain right now. Some of the counts may be inaccurate."
 msgstr ""
@@ -3254,11 +3254,6 @@ msgstr ""
 msgid "truffle flattener"
 msgstr ""
 
-#: lib/block_scout_web/templates/layout/app.html.eex:44
-#, elixir-autogen, elixir-format, fuzzy
-msgid "Indexing Internal Transactions"
-msgstr ""
-
 #: lib/block_scout_web/templates/address_contract_verification_common_fields/_library_first.html.eex:5
 #, elixir-autogen, elixir-format
 msgid ") may be added for each contract. Click the Add Library button to add an additional one."
@@ -3424,3 +3419,8 @@ msgstr ""
 #, elixir-autogen, elixir-format, fuzzy
 msgid "Verifed contracts, %{subnetwork}, %{coin}"
 msgstr ""
+
+#: lib/block_scout_web/templates/layout/app.html.eex:44
+#, elixir-autogen, elixir-format, fuzzy
+msgid "Blocks With Internal Transactions Indexed"
+msgstr ""

apps/block_scout_web/test/block_scout_web/features/viewing_app_test.exs

@@ -27,7 +27,7 @@ defmodule BlockScoutWeb.ViewingAppTest do
   #     |> insert()
   #     |> with_block(block)
 
-  #     assert Decimal.compare(Explorer.Chain.indexed_ratio(), Decimal.from_float(0.5)) == :eq
+  #     assert Decimal.compare(Explorer.Chain.indexed_ratio_blocks(), Decimal.from_float(0.5)) == :eq
 
   #     insert(:pending_block_operation, block_hash: block.hash, fetch_internal_transactions: true)
 
@@ -46,7 +46,7 @@ defmodule BlockScoutWeb.ViewingAppTest do
   #     |> insert()
   #     |> with_block(block)
 
-  #     assert Decimal.compare(Explorer.Chain.indexed_ratio(), 1) == :eq
+  #     assert Decimal.compare(Explorer.Chain.indexed_ratio_blocks(), 1) == :eq
 
   #     insert(:pending_block_operation, block_hash: block.hash, fetch_internal_transactions: true)
 
@@ -67,7 +67,7 @@ defmodule BlockScoutWeb.ViewingAppTest do
 
   #     BlocksIndexedCounter.calculate_blocks_indexed()
 
-  #     assert Decimal.compare(Explorer.Chain.indexed_ratio(), Decimal.from_float(0.5)) == :eq
+  #     assert Decimal.compare(Explorer.Chain.indexed_ratio_blocks(), Decimal.from_float(0.5)) == :eq
 
   #     insert(:pending_block_operation, block_hash: block.hash, fetch_internal_transactions: true)
 
@@ -125,7 +125,7 @@ defmodule BlockScoutWeb.ViewingAppTest do
 
   #     BlocksIndexedCounter.calculate_blocks_indexed()
 
-  #     assert Decimal.compare(Explorer.Chain.indexed_ratio(), 1) == :eq
+  #     assert Decimal.compare(Explorer.Chain.indexed_ratio_blocks(), 1) == :eq
 
   #     session
   #     |> AppPage.visit_page()

apps/explorer/lib/explorer/chain.ex

@@ -1258,13 +1258,17 @@ defmodule Explorer.Chain do
     end
   end
 
+  def finished_blocks_indexing?(indexed_ratio_blocks) do
+    Decimal.compare(indexed_ratio_blocks, 1) !== :lt
+  end
+
   @doc """
   Checks if indexing of blocks and internal transactions finished aka full indexing
   """
   @spec finished_indexing?(Decimal.t()) :: boolean()
-  def finished_indexing?(indexed_ratio) do
-    case Decimal.compare(indexed_ratio, 1) do
-      :lt -> false
+  def finished_indexing?(indexed_ratio_blocks) do
+    case finished_blocks_indexing?(indexed_ratio_blocks) do
+      false -> false
       _ -> Chain.finished_internal_transactions_indexing?()
     end
   end
@@ -2158,17 +2162,17 @@ defmodule Explorer.Chain do
       ...>   insert(:block, number: index)
       ...>   Process.sleep(200)
       ...> end
-      iex> Explorer.Chain.indexed_ratio()
+      iex> Explorer.Chain.indexed_ratio_blocks()
       Decimal.new(1, 50, -2)
 
   If there are no blocks, the percentage is 0.
 
-      iex> Explorer.Chain.indexed_ratio()
+      iex> Explorer.Chain.indexed_ratio_blocks()
       Decimal.new(0)
 
   """
-  @spec indexed_ratio() :: Decimal.t()
-  def indexed_ratio do
+  @spec indexed_ratio_blocks() :: Decimal.t()
+  def indexed_ratio_blocks do
     %{min: min, max: max} = BlockNumber.get_all()
 
     min_blockchain_block_number =
@@ -2190,6 +2194,31 @@ defmodule Explorer.Chain do
     end
   end
 
+  @spec indexed_ratio_internal_transactions() :: Decimal.t()
+  def indexed_ratio_internal_transactions do
+    %{max: max} = BlockNumber.get_all()
+    count = Repo.aggregate(PendingBlockOperation, :count, timeout: :infinity)
+
+    min_blockchain_trace_block_number =
+      case Integer.parse(Application.get_env(:indexer, :trace_first_block)) do
+        {block_number, _} -> block_number
+        _ -> 0
+      end
+
+    case max do
+      0 ->
+        Decimal.new(0)
+
+      _ ->
+        full_blocks_range = max - min_blockchain_trace_block_number + 1
+        result = Decimal.div(full_blocks_range - count, full_blocks_range)
+
+        result
+        |> Decimal.round(2, :down)
+        |> Decimal.min(Decimal.new(1))
+    end
+  end
+
   @spec fetch_min_block_number() :: non_neg_integer
   def fetch_min_block_number do
     query =

apps/explorer/test/explorer/chain_test.exs

@@ -1466,7 +1466,7 @@ defmodule Explorer.ChainTest do
     end
   end
 
-  describe "indexed_ratio/0" do
+  describe "indexed_ratio_blocks/0" do
     setup do
       on_exit(fn ->
         Application.put_env(:indexer, :first_block, "")
@@ -1478,11 +1478,11 @@ defmodule Explorer.ChainTest do
         insert(:block, number: index)
       end
 
-      assert Decimal.compare(Chain.indexed_ratio(), Decimal.from_float(0.5)) == :eq
+      assert Decimal.compare(Chain.indexed_ratio_blocks(), Decimal.from_float(0.5)) == :eq
     end
 
     test "returns 0 if no blocks" do
-      assert Decimal.new(0) == Chain.indexed_ratio()
+      assert Decimal.new(0) == Chain.indexed_ratio_blocks()
     end
 
     test "returns 1.0 if fully indexed blocks" do
@@ -1491,7 +1491,7 @@ defmodule Explorer.ChainTest do
         Process.sleep(200)
       end
 
-      assert Decimal.compare(Chain.indexed_ratio(), 1) == :eq
+      assert Decimal.compare(Chain.indexed_ratio_blocks(), 1) == :eq
     end
 
     test "returns 1.0 if fully indexed blocks starting from given FIRST_BLOCK" do
@@ -1502,7 +1502,49 @@ defmodule Explorer.ChainTest do
         Process.sleep(200)
       end
 
-      assert Decimal.compare(Chain.indexed_ratio(), 1) == :eq
+      assert Decimal.compare(Chain.indexed_ratio_blocks(), 1) == :eq
+    end
+  end
+
+  describe "indexed_ratio_internal_transactions/0" do
+    setup do
+      on_exit(fn ->
+        Application.put_env(:indexer, :trace_first_block, "")
+      end)
+    end
+
+    test "returns indexed ratio" do
+      for index <- 0..9 do
+        block = insert(:block, number: index)
+
+        if index === 0 || index === 5 || index === 7 do
+          insert(:pending_block_operation, block: block, fetch_internal_transactions: true)
+        end
+      end
+
+      assert Decimal.compare(Chain.indexed_ratio_internal_transactions(), Decimal.from_float(0.7)) == :eq
+    end
+
+    test "returns 0 if no blocks" do
+      assert Decimal.new(0) == Chain.indexed_ratio_internal_transactions()
+    end
+
+    test "returns 1.0 if no pending block operations" do
+      for index <- 0..9 do
+        insert(:block, number: index)
+      end
+
+      assert Decimal.compare(Chain.indexed_ratio_internal_transactions(), 1) == :eq
+    end
+
+    test "returns 1.0 if fully indexed blocks with internal transactions starting from given TRACE_FIRST_BLOCK" do
+      Application.put_env(:indexer, :trace_first_block, "5")
+
+      for index <- 5..9 do
+        insert(:block, number: index)
+      end
+
+      assert Decimal.compare(Chain.indexed_ratio_internal_transactions(), 1) == :eq
     end
   end