Denormalization improvements

.dialyzer-ignore

@@ -23,4 +23,4 @@ lib/indexer/fetcher/zkevm/transaction_batch.ex:156
 lib/indexer/fetcher/zkevm/transaction_batch.ex:252
 lib/block_scout_web/views/api/v2/transaction_view.ex:431
 lib/block_scout_web/views/api/v2/transaction_view.ex:472
-lib/explorer/chain/transaction.ex:169
+lib/explorer/chain/transaction.ex:170

apps/explorer/lib/explorer/chain.ex

@@ -353,15 +353,26 @@ defmodule Explorer.Chain do
     to_block = to_block(options)
 
     base =
-      from(log in Log,
-        order_by: [desc: log.block_number, desc: log.index],
-        where: log.address_hash == ^address_hash,
-        limit: ^paging_options.page_size,
-        select: log,
-        inner_join: block in Block,
-        on: block.hash == log.block_hash,
-        where: block.consensus == true
-      )
+      if DenormalizationHelper.denormalization_finished?() do
+        from(log in Log,
+          order_by: [desc: log.block_number, desc: log.index],
+          where: log.address_hash == ^address_hash,
+          limit: ^paging_options.page_size,
+          select: log,
+          inner_join: transaction in assoc(log, :transaction),
+          where: transaction.block_consensus == true
+        )
+      else
+        from(log in Log,
+          order_by: [desc: log.block_number, desc: log.index],
+          where: log.address_hash == ^address_hash,
+          limit: ^paging_options.page_size,
+          select: log,
+          inner_join: block in Block,
+          on: block.hash == log.block_hash,
+          where: block.consensus == true
+        )
+      end
 
     preloaded_query =
       if csv_export? do

apps/explorer/lib/explorer/chain/cache/gas_price_oracle.ex

@@ -14,6 +14,8 @@ defmodule Explorer.Chain.Cache.GasPriceOracle do
 
   alias Explorer.Chain.{
     Block,
+    DenormalizationHelper,
+    Transaction,
     Wei
   }
 
@@ -73,77 +75,150 @@ defmodule Explorer.Chain.Cache.GasPriceOracle do
       end
 
     fee_query =
-      from(
-        block in Block,
-        left_join: transaction in assoc(block, :transactions),
-        where: block.consensus == true,
-        where: transaction.status == ^1,
-        where: transaction.gas_price > ^0,
-        where: transaction.block_number > ^from_block,
-        group_by: transaction.block_number,
-        order_by: [desc: transaction.block_number],
-        select: %{
-          block_number: transaction.block_number,
-          slow_gas_price:
-            fragment(
-              "percentile_disc(? :: real) within group ( order by ? )",
-              ^safelow_percentile_fraction,
-              transaction.gas_price
-            ),
-          average_gas_price:
-            fragment(
-              "percentile_disc(? :: real) within group ( order by ? )",
-              ^average_percentile_fraction,
-              transaction.gas_price
-            ),
-          fast_gas_price:
-            fragment(
-              "percentile_disc(? :: real) within group ( order by ? )",
-              ^fast_percentile_fraction,
-              transaction.gas_price
-            ),
-          slow_priority_fee_per_gas:
-            fragment(
-              "percentile_disc(? :: real) within group ( order by ? )",
-              ^safelow_percentile_fraction,
-              transaction.max_priority_fee_per_gas
-            ),
-          average_priority_fee_per_gas:
-            fragment(
-              "percentile_disc(? :: real) within group ( order by ? )",
-              ^average_percentile_fraction,
-              transaction.max_priority_fee_per_gas
-            ),
-          fast_priority_fee_per_gas:
-            fragment(
-              "percentile_disc(? :: real) within group ( order by ? )",
-              ^fast_percentile_fraction,
-              transaction.max_priority_fee_per_gas
-            ),
-          slow_time:
-            fragment(
-              "percentile_disc(? :: real) within group ( order by coalesce(extract(milliseconds from (?)::interval), ?) desc )",
-              ^safelow_percentile_fraction,
-              block.timestamp - transaction.earliest_processing_start,
-              ^average_block_time
-            ),
-          average_time:
-            fragment(
-              "percentile_disc(? :: real) within group ( order by coalesce(extract(milliseconds from (?)::interval), ?) desc )",
-              ^average_percentile_fraction,
-              block.timestamp - transaction.earliest_processing_start,
-              ^average_block_time
-            ),
-          fast_time:
-            fragment(
-              "percentile_disc(? :: real) within group ( order by coalesce(extract(milliseconds from (?)::interval), ?) desc )",
-              ^fast_percentile_fraction,
-              block.timestamp - transaction.earliest_processing_start,
-              ^average_block_time
-            )
-        },
-        limit: ^num_of_blocks
-      )
+      if DenormalizationHelper.denormalization_finished?() do
+        from(
+          transaction in Transaction,
+          where: transaction.block_consensus == true,
+          where: transaction.status == ^1,
+          where: transaction.gas_price > ^0,
+          where: transaction.block_number > ^from_block,
+          group_by: transaction.block_number,
+          order_by: [desc: transaction.block_number],
+          select: %{
+            block_number: transaction.block_number,
+            slow_gas_price:
+              fragment(
+                "percentile_disc(? :: real) within group ( order by ? )",
+                ^safelow_percentile_fraction,
+                transaction.gas_price
+              ),
+            average_gas_price:
+              fragment(
+                "percentile_disc(? :: real) within group ( order by ? )",
+                ^average_percentile_fraction,
+                transaction.gas_price
+              ),
+            fast_gas_price:
+              fragment(
+                "percentile_disc(? :: real) within group ( order by ? )",
+                ^fast_percentile_fraction,
+                transaction.gas_price
+              ),
+            slow_priority_fee_per_gas:
+              fragment(
+                "percentile_disc(? :: real) within group ( order by ? )",
+                ^safelow_percentile_fraction,
+                transaction.max_priority_fee_per_gas
+              ),
+            average_priority_fee_per_gas:
+              fragment(
+                "percentile_disc(? :: real) within group ( order by ? )",
+                ^average_percentile_fraction,
+                transaction.max_priority_fee_per_gas
+              ),
+            fast_priority_fee_per_gas:
+              fragment(
+                "percentile_disc(? :: real) within group ( order by ? )",
+                ^fast_percentile_fraction,
+                transaction.max_priority_fee_per_gas
+              ),
+            slow_time:
+              fragment(
+                "percentile_disc(? :: real) within group ( order by coalesce(extract(milliseconds from (?)::interval), ?) desc )",
+                ^safelow_percentile_fraction,
+                transaction.block_timestamp - transaction.earliest_processing_start,
+                ^average_block_time
+              ),
+            average_time:
+              fragment(
+                "percentile_disc(? :: real) within group ( order by coalesce(extract(milliseconds from (?)::interval), ?) desc )",
+                ^average_percentile_fraction,
+                transaction.block_timestamp - transaction.earliest_processing_start,
+                ^average_block_time
+              ),
+            fast_time:
+              fragment(
+                "percentile_disc(? :: real) within group ( order by coalesce(extract(milliseconds from (?)::interval), ?) desc )",
+                ^fast_percentile_fraction,
+                transaction.block_timestamp - transaction.earliest_processing_start,
+                ^average_block_time
+              )
+          },
+          limit: ^num_of_blocks
+        )
+      else
+        from(
+          block in Block,
+          left_join: transaction in assoc(block, :transactions),
+          where: block.consensus == true,
+          where: transaction.status == ^1,
+          where: transaction.gas_price > ^0,
+          where: transaction.block_number > ^from_block,
+          group_by: transaction.block_number,
+          order_by: [desc: transaction.block_number],
+          select: %{
+            block_number: transaction.block_number,
+            slow_gas_price:
+              fragment(
+                "percentile_disc(? :: real) within group ( order by ? )",
+                ^safelow_percentile_fraction,
+                transaction.gas_price
+              ),
+            average_gas_price:
+              fragment(
+                "percentile_disc(? :: real) within group ( order by ? )",
+                ^average_percentile_fraction,
+                transaction.gas_price
+              ),
+            fast_gas_price:
+              fragment(
+                "percentile_disc(? :: real) within group ( order by ? )",
+                ^fast_percentile_fraction,
+                transaction.gas_price
+              ),
+            slow_priority_fee_per_gas:
+              fragment(
+                "percentile_disc(? :: real) within group ( order by ? )",
+                ^safelow_percentile_fraction,
+                transaction.max_priority_fee_per_gas
+              ),
+            average_priority_fee_per_gas:
+              fragment(
+                "percentile_disc(? :: real) within group ( order by ? )",
+                ^average_percentile_fraction,
+                transaction.max_priority_fee_per_gas
+              ),
+            fast_priority_fee_per_gas:
+              fragment(
+                "percentile_disc(? :: real) within group ( order by ? )",
+                ^fast_percentile_fraction,
+                transaction.max_priority_fee_per_gas
+              ),
+            slow_time:
+              fragment(
+                "percentile_disc(? :: real) within group ( order by coalesce(extract(milliseconds from (?)::interval), ?) desc )",
+                ^safelow_percentile_fraction,
+                block.timestamp - transaction.earliest_processing_start,
+                ^average_block_time
+              ),
+            average_time:
+              fragment(
+                "percentile_disc(? :: real) within group ( order by coalesce(extract(milliseconds from (?)::interval), ?) desc )",
+                ^average_percentile_fraction,
+                block.timestamp - transaction.earliest_processing_start,
+                ^average_block_time
+              ),
+            fast_time:
+              fragment(
+                "percentile_disc(? :: real) within group ( order by coalesce(extract(milliseconds from (?)::interval), ?) desc )",
+                ^fast_percentile_fraction,
+                block.timestamp - transaction.earliest_processing_start,
+                ^average_block_time
+              )
+          },
+          limit: ^num_of_blocks
+        )
+      end
 
     new_acc = fee_query |> Repo.all(timeout: :infinity) |> merge_gas_prices(acc, num_of_blocks)
 

apps/explorer/lib/explorer/chain/denormalization_helper.ex

@@ -5,6 +5,7 @@ defmodule Explorer.Chain.DenormalizationHelper do
 
   alias Explorer.Chain.Cache.BackgroundMigrations
 
+  @spec extend_block_necessity(keyword(), :optional | :required) :: keyword()
   def extend_block_necessity(opts, necessity \\ :optional) do
     if denormalization_finished?() do
       opts
@@ -13,6 +14,7 @@ defmodule Explorer.Chain.DenormalizationHelper do
     end
   end
 
+  @spec extend_transaction_block_necessity(keyword(), :optional | :required) :: keyword()
   def extend_transaction_block_necessity(opts, necessity \\ :optional) do
     if denormalization_finished?() do
       opts
@@ -26,6 +28,7 @@ defmodule Explorer.Chain.DenormalizationHelper do
     end
   end
 
+  @spec extend_transaction_preload(list()) :: list()
   def extend_transaction_preload(preloads) do
     if denormalization_finished?() do
       preloads
@@ -34,6 +37,7 @@ defmodule Explorer.Chain.DenormalizationHelper do
     end
   end
 
+  @spec extend_block_preload(list()) :: list()
   def extend_block_preload(preloads) do
     if denormalization_finished?() do
       preloads

apps/explorer/lib/explorer/utility/migration_status.ex

@@ -9,6 +9,7 @@ defmodule Explorer.Utility.MigrationStatus do
   @primary_key false
   schema "migrations_status" do
     field(:migration_name, :string)
+    # ["started", "completed"]
     field(:status, :string)
 
     timestamps()

docker-compose/envs/common-blockscout.env

@@ -267,4 +267,6 @@ API_V2_ENABLED=true
 # ACCOUNT_PRIVATE_TAGS_LIMIT=2000
 # ACCOUNT_WATCHLIST_ADDRESSES_LIMIT=15
 # MICROSERVICE_BENS_URL=
-# MICROSERVICE_BENS_ENABLED=
+# MICROSERVICE_BENS_ENABLED=
+# DENORMALIZATION_MIGRATION_BATCH_SIZE=
+# DENORMALIZATION_MIGRATION_CONCURRENCY=