Null round handling (#9403)

* Null round handling

* Add repo for filecoin chain type

* Add repo for filecoin chain type

* Modify gas price constraint for Filecoin as it for PolygonEdge

* Fix null round heights db type

* Add filecoin to chain-type matrix

---------

Co-authored-by: Viktor Baranov <baranov.viktor.27@gmail.com>
dependabot/hex/briefly-a533393
Qwerty5Uiop 8 months ago committed by GitHub
parent 15f708e1fd
commit 42425edef8
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
  1. 2
      .github/workflows/config.yml
  2. 2
      .github/workflows/publish-docker-image-for-filecoin.yml
  3. 1
      CHANGELOG.md
  4. 12
      apps/block_scout_web/lib/block_scout_web/models/transaction_state_helper.ex
  5. 9
      apps/block_scout_web/lib/block_scout_web/notifier.ex
  6. 1
      apps/block_scout_web/test/test_helper.exs
  7. 2
      apps/explorer/config/dev.exs
  8. 4
      apps/explorer/config/prod.exs
  9. 3
      apps/explorer/config/test.exs
  10. 3
      apps/explorer/lib/explorer/application.ex
  11. 75
      apps/explorer/lib/explorer/chain.ex
  12. 25
      apps/explorer/lib/explorer/chain/block_number_helper.ex
  13. 8
      apps/explorer/lib/explorer/chain/import/runner/blocks.ex
  14. 81
      apps/explorer/lib/explorer/chain/null_round_height.ex
  15. 10
      apps/explorer/lib/explorer/repo.ex
  16. 21
      apps/explorer/lib/explorer/utility/missing_block_range.ex
  17. 15
      apps/explorer/priv/filecoin/migrations/20230731130103_modify_collated_gas_price_constraint.exs
  18. 9
      apps/explorer/priv/filecoin/migrations/20231109104957_create_null_round_heights.exs
  19. 9
      apps/explorer/priv/filecoin/migrations/20240219140124_change_null_round_heights_height_type.exs
  20. 1
      apps/explorer/test/test_helper.exs
  21. 18
      apps/indexer/lib/indexer/block/catchup/fetcher.ex
  22. 4
      apps/indexer/lib/indexer/fetcher/transaction_action.ex
  23. 1
      config/config_helper.exs
  24. 7
      config/runtime/dev.exs
  25. 6
      config/runtime/prod.exs
  26. 4
      cspell.json

@ -48,7 +48,7 @@ jobs:
run: |
echo "matrix=$matrixStringifiedObject" >> $GITHUB_OUTPUT
env:
matrixStringifiedObject: '{"chain-type": ["ethereum", "polygon_edge", "polygon_zkevm", "rsk", "suave", "stability"]}'
matrixStringifiedObject: '{"chain-type": ["ethereum", "polygon_edge", "polygon_zkevm", "rsk", "suave", "stability", "filecoin"]}'
build-and-cache:
name: Build and Cache deps

@ -36,4 +36,4 @@ jobs:
CACHE_ADDRESS_WITH_BALANCES_UPDATE_INTERVAL=
BLOCKSCOUT_VERSION=v${{ env.RELEASE_VERSION }}-beta.+commit.${{ env.SHORT_SHA }}
RELEASE_VERSION=${{ env.RELEASE_VERSION }}
CHAIN_TYPE=polygon_edge
CHAIN_TYPE=filecoin

@ -4,6 +4,7 @@
### Features
- [#9403](https://github.com/blockscout/blockscout/pull/9403) - Null round handling
- [#9396](https://github.com/blockscout/blockscout/pull/9396) - More-Minimal Proxy support
- [#9379](https://github.com/blockscout/blockscout/pull/9379) - Filter non-traceable transactions for zetachain
- [#9364](https://github.com/blockscout/blockscout/pull/9364) - Fix using of startblock/endblock in API v1 list endpoints: txlist, txlistinternal, tokentx

@ -8,7 +8,7 @@ defmodule BlockScoutWeb.Models.TransactionStateHelper do
alias Explorer.Chain.Transaction.StateChange
alias Explorer.{Chain, PagingOptions}
alias Explorer.Chain.{Block, Transaction, Wei}
alias Explorer.Chain.{Block, BlockNumberHelper, Transaction, Wei}
alias Explorer.Chain.Cache.StateChanges
alias Indexer.Fetcher.{CoinBalanceOnDemand, TokenBalanceOnDemand}
@ -73,9 +73,11 @@ defmodule BlockScoutWeb.Models.TransactionStateHelper do
api?: Keyword.get(options, :api?, false)
)
from_before_block = coin_balance(transaction.from_address_hash, block.number - 1, options)
to_before_block = coin_balance(transaction.to_address_hash, block.number - 1, options)
miner_before_block = coin_balance(block.miner_hash, block.number - 1, options)
previous_block_number = BlockNumberHelper.previous_block_number(block.number)
from_before_block = coin_balance(transaction.from_address_hash, previous_block_number, options)
to_before_block = coin_balance(transaction.to_address_hash, previous_block_number, options)
miner_before_block = coin_balance(block.miner_hash, previous_block_number, options)
{from_before_tx, to_before_tx, miner_before_tx} =
StateChange.coin_balances_before(transaction, block_txs, from_before_block, to_before_block, miner_before_block)
@ -146,7 +148,7 @@ defmodule BlockScoutWeb.Models.TransactionStateHelper do
from = transfer.from_address
to = transfer.to_address
token_hash = transfer.token_contract_address_hash
prev_block = transfer.block_number - 1
prev_block = BlockNumberHelper.previous_block_number(transfer.block_number)
balances
|> case do

@ -20,7 +20,7 @@ defmodule BlockScoutWeb.Notifier do
alias Explorer.{Chain, Market, Repo}
alias Explorer.Chain.Address.Counters
alias Explorer.Chain.{Address, DenormalizationHelper, InternalTransaction, Transaction}
alias Explorer.Chain.{Address, BlockNumberHelper, DenormalizationHelper, InternalTransaction, Transaction}
alias Explorer.Chain.Supply.RSK
alias Explorer.Chain.Transaction.History.TransactionStats
alias Explorer.Counters.{AverageBlockTime, Helper}
@ -305,12 +305,13 @@ defmodule BlockScoutWeb.Notifier do
defp broadcast_latest_block?(block, last_broadcasted_block_number) do
cond do
last_broadcasted_block_number == 0 || last_broadcasted_block_number == block.number - 1 ||
last_broadcasted_block_number == 0 ||
last_broadcasted_block_number == BlockNumberHelper.previous_block_number(block.number) ||
last_broadcasted_block_number < block.number - 4 ->
broadcast_block(block)
:ets.insert(:last_broadcasted_block, {:number, block.number})
last_broadcasted_block_number > block.number - 1 ->
last_broadcasted_block_number > BlockNumberHelper.previous_block_number(block.number) ->
broadcast_block(block)
true ->
@ -324,7 +325,7 @@ defmodule BlockScoutWeb.Notifier do
:timer.sleep(@check_broadcast_sequence_period)
last_broadcasted_block_number = Helper.fetch_from_cache(:number, :last_broadcasted_block)
if last_broadcasted_block_number == block.number - 1 do
if last_broadcasted_block_number == BlockNumberHelper.previous_block_number(block.number) do
broadcast_block(block)
:ets.insert(:last_broadcasted_block, {:number, block.number})
else

@ -33,6 +33,7 @@ Ecto.Adapters.SQL.Sandbox.mode(Explorer.Repo.Shibarium, :manual)
Ecto.Adapters.SQL.Sandbox.mode(Explorer.Repo.Suave, :manual)
Ecto.Adapters.SQL.Sandbox.mode(Explorer.Repo.Beacon, :manual)
Ecto.Adapters.SQL.Sandbox.mode(Explorer.Repo.BridgedTokens, :manual)
Ecto.Adapters.SQL.Sandbox.mode(Explorer.Repo.Filecoin, :manual)
Absinthe.Test.prime(BlockScoutWeb.Schema)

@ -27,6 +27,8 @@ config :explorer, Explorer.Repo.Beacon, timeout: :timer.seconds(80)
config :explorer, Explorer.Repo.BridgedTokens, timeout: :timer.seconds(80)
config :explorer, Explorer.Repo.Filecoin, timeout: :timer.seconds(80)
config :explorer, Explorer.Tracer, env: "dev", disabled?: true
config :logger, :explorer,

@ -44,6 +44,10 @@ config :explorer, Explorer.Repo.BridgedTokens,
prepare: :unnamed,
timeout: :timer.seconds(60)
config :explorer, Explorer.Repo.Filecoin,
prepare: :unnamed,
timeout: :timer.seconds(60)
config :explorer, Explorer.Tracer, env: "production", disabled?: true
config :logger, :explorer,

@ -50,7 +50,8 @@ for repo <- [
Explorer.Repo.RSK,
Explorer.Repo.Shibarium,
Explorer.Repo.Suave,
Explorer.Repo.BridgedTokens
Explorer.Repo.BridgedTokens,
Explorer.Repo.Filecoin
] do
config :explorer, repo,
database: "explorer_test",

@ -145,7 +145,8 @@ defmodule Explorer.Application do
Explorer.Repo.RSK,
Explorer.Repo.Shibarium,
Explorer.Repo.Suave,
Explorer.Repo.BridgedTokens
Explorer.Repo.BridgedTokens,
Explorer.Repo.Filecoin
]
else
[]

@ -48,6 +48,7 @@ defmodule Explorer.Chain do
Address.CurrentTokenBalance,
Address.TokenBalance,
Block,
BlockNumberHelper,
CurrencyHelper,
Data,
DecompiledSmartContract,
@ -1426,7 +1427,7 @@ defmodule Explorer.Chain do
Decimal.new(1)
_ ->
divisor = max_saved_block_number - min_blockchain_block_number + 1
divisor = max_saved_block_number - min_blockchain_block_number - BlockNumberHelper.null_rounds_count() + 1
ratio = get_ratio(BlockCache.estimated_count(), divisor)
@ -1458,7 +1459,9 @@ defmodule Explorer.Chain do
Decimal.new(0)
_ ->
full_blocks_range = max_saved_block_number - min_blockchain_trace_block_number + 1
full_blocks_range =
max_saved_block_number - min_blockchain_trace_block_number - BlockNumberHelper.null_rounds_count() + 1
processed_int_txs_for_blocks_count = max(0, full_blocks_range - pbo_count)
ratio = get_ratio(processed_int_txs_for_blocks_count, full_blocks_range)
@ -2211,26 +2214,50 @@ defmodule Explorer.Chain do
range_max = max(range_start, range_end)
ordered_missing_query =
from(b in Block,
right_join:
missing_range in fragment(
"""
(
SELECT distinct b1.number
FROM generate_series((?)::integer, (?)::integer) AS b1(number)
WHERE NOT EXISTS
(SELECT 1 FROM blocks b2 WHERE b2.number=b1.number AND b2.consensus)
ORDER BY b1.number DESC
)
""",
^range_min,
^range_max
),
on: b.number == missing_range.number,
select: missing_range.number,
order_by: missing_range.number,
distinct: missing_range.number
)
if Application.get_env(:explorer, :chain_type) == "filecoin" do
from(b in Block,
right_join:
missing_range in fragment(
"""
(
SELECT distinct b1.number
FROM generate_series((?)::integer, (?)::integer) AS b1(number)
WHERE NOT EXISTS
(SELECT 1 FROM blocks b2 WHERE b2.number=b1.number AND b2.consensus)
AND NOT EXISTS (SELECT 1 FROM null_round_heights nrh where nrh.height=b1.number)
ORDER BY b1.number DESC
)
""",
^range_min,
^range_max
),
on: b.number == missing_range.number,
select: missing_range.number,
order_by: missing_range.number,
distinct: missing_range.number
)
else
from(b in Block,
right_join:
missing_range in fragment(
"""
(
SELECT distinct b1.number
FROM generate_series((?)::integer, (?)::integer) AS b1(number)
WHERE NOT EXISTS
(SELECT 1 FROM blocks b2 WHERE b2.number=b1.number AND b2.consensus)
ORDER BY b1.number DESC
)
""",
^range_min,
^range_max
),
on: b.number == missing_range.number,
select: missing_range.number,
order_by: missing_range.number,
distinct: missing_range.number
)
end
missing_blocks = Repo.all(ordered_missing_query, timeout: :infinity)
@ -2368,13 +2395,13 @@ defmodule Explorer.Chain do
DateTime.compare(timestamp, given_timestamp) == :eq do
number
else
number - 1
BlockNumberHelper.previous_block_number(number)
end
:after ->
if DateTime.compare(timestamp, given_timestamp) == :lt ||
DateTime.compare(timestamp, given_timestamp) == :eq do
number + 1
BlockNumberHelper.next_block_number(number)
else
number
end

@ -0,0 +1,25 @@
# credo:disable-for-this-file
defmodule Explorer.Chain.BlockNumberHelper do
@moduledoc """
Functions to operate with block numbers based on null round heights (applicable for CHAIN_TYPE=filecoin)
"""
def previous_block_number(number), do: neighbor_block_number(number, :previous)
def next_block_number(number), do: neighbor_block_number(number, :next)
case Application.compile_env(:explorer, :chain_type) do
"filecoin" ->
def null_rounds_count, do: Explorer.Chain.NullRoundHeight.total()
defp neighbor_block_number(number, direction),
do: Explorer.Chain.NullRoundHeight.neighbor_block_number(number, direction)
_ ->
def null_rounds_count, do: 0
defp neighbor_block_number(number, direction), do: move_by_one(number, direction)
end
def move_by_one(number, :previous), do: number - 1
def move_by_one(number, :next), do: number + 1
end

@ -14,6 +14,7 @@ defmodule Explorer.Chain.Import.Runner.Blocks do
alias Explorer.Chain.{
Address,
Block,
BlockNumberHelper,
Import,
PendingBlockOperation,
Token,
@ -884,11 +885,14 @@ defmodule Explorer.Chain.Import.Runner.Blocks do
number: number
},
acc ->
previous_block_number = BlockNumberHelper.previous_block_number(number)
next_block_number = BlockNumberHelper.next_block_number(number)
if consensus do
from(
block in acc,
or_where: block.number == ^(number - 1) and block.hash != ^parent_hash,
or_where: block.number == ^(number + 1) and block.parent_hash != ^hash
or_where: block.number == ^previous_block_number and block.hash != ^parent_hash,
or_where: block.number == ^next_block_number and block.parent_hash != ^hash
)
else
acc

@ -0,0 +1,81 @@
defmodule Explorer.Chain.NullRoundHeight do
@moduledoc """
A null round is formed when a block at height N links to a block at height N-2 instead of N-1
"""
use Explorer.Schema
alias Explorer.Repo
@primary_key false
schema "null_round_heights" do
field(:height, :integer, primary_key: true)
end
def changeset(null_round_height \\ %__MODULE__{}, params) do
null_round_height
|> cast(params, [:height])
|> validate_required([:height])
|> unique_constraint(:height)
end
def total do
Repo.aggregate(__MODULE__, :count)
end
def insert_heights(heights) do
params =
heights
|> Enum.uniq()
|> Enum.map(&%{height: &1})
Repo.insert_all(__MODULE__, params, on_conflict: :nothing)
end
defp find_neighbor_from_previous(previous_null_rounds, number, direction) do
previous_null_rounds
|> Enum.reduce_while({number, nil}, fn height, {current, _result} ->
if height == move_by_one(current, direction) do
{:cont, {height, nil}}
else
{:halt, {nil, move_by_one(current, direction)}}
end
end)
|> elem(1)
|> case do
nil ->
previous_null_rounds
|> List.last()
|> neighbor_block_number(direction)
number ->
number
end
end
def neighbor_block_number(number, direction) do
number
|> neighbors_query(direction)
|> select([nrh], nrh.height)
|> Repo.all()
|> case do
[] ->
move_by_one(number, direction)
previous_null_rounds ->
find_neighbor_from_previous(previous_null_rounds, number, direction)
end
end
defp move_by_one(number, :previous), do: number - 1
defp move_by_one(number, :next), do: number + 1
@batch_size 5
defp neighbors_query(number, :previous) do
from(nrh in __MODULE__, where: nrh.height < ^number, order_by: [desc: :height], limit: @batch_size)
end
defp neighbors_query(number, :next) do
from(nrh in __MODULE__, where: nrh.height > ^number, order_by: [asc: :height], limit: @batch_size)
end
end

@ -220,4 +220,14 @@ defmodule Explorer.Repo do
ConfigHelper.init_repo_module(__MODULE__, opts)
end
end
defmodule Filecoin do
use Ecto.Repo,
otp_app: :explorer,
adapter: Ecto.Adapters.Postgres
def init(_, opts) do
ConfigHelper.init_repo_module(__MODULE__, opts)
end
end
end

@ -4,6 +4,7 @@ defmodule Explorer.Utility.MissingBlockRange do
"""
use Explorer.Schema
alias Explorer.Chain.BlockNumberHelper
alias Explorer.Repo
@default_returning_batch_size 10
@ -76,21 +77,29 @@ defmodule Explorer.Utility.MissingBlockRange do
case {lower_range, higher_range} do
{%__MODULE__{} = same_range, %__MODULE__{} = same_range} ->
Repo.delete(same_range)
insert_if_needed(%{from_number: same_range.from_number, to_number: max_number + 1})
insert_if_needed(%{from_number: min_number - 1, to_number: same_range.to_number})
insert_if_needed(%{
from_number: same_range.from_number,
to_number: BlockNumberHelper.next_block_number(max_number)
})
insert_if_needed(%{
from_number: BlockNumberHelper.previous_block_number(min_number),
to_number: same_range.to_number
})
{%__MODULE__{} = range, nil} ->
delete_ranges_between(max_number, range.from_number)
update_from_number_or_delete_range(range, min_number - 1)
update_from_number_or_delete_range(range, BlockNumberHelper.previous_block_number(min_number))
{nil, %__MODULE__{} = range} ->
delete_ranges_between(range.to_number, min_number)
update_to_number_or_delete_range(range, max_number + 1)
update_to_number_or_delete_range(range, BlockNumberHelper.next_block_number(max_number))
{%__MODULE__{} = range_1, %__MODULE__{} = range_2} ->
delete_ranges_between(range_2.to_number, range_1.from_number)
update_from_number_or_delete_range(range_1, min_number - 1)
update_to_number_or_delete_range(range_2, max_number + 1)
update_from_number_or_delete_range(range_1, BlockNumberHelper.previous_block_number(min_number))
update_to_number_or_delete_range(range_2, BlockNumberHelper.next_block_number(max_number))
_ ->
delete_ranges_between(max_number, min_number)

@ -0,0 +1,15 @@
defmodule Explorer.Repo.Filecoin.Migrations.ModifyCollatedGasPriceConstraint do
use Ecto.Migration
def change do
execute("ALTER TABLE transactions DROP CONSTRAINT collated_gas_price")
create(
constraint(
:transactions,
:collated_gas_price,
check: "block_hash IS NULL OR gas_price IS NOT NULL OR max_fee_per_gas IS NOT NULL"
)
)
end
end

@ -0,0 +1,9 @@
defmodule Explorer.Repo.Filecoin.Migrations.CreateNullRoundHeights do
use Ecto.Migration
def change do
create table(:null_round_heights, primary_key: false) do
add(:height, :integer, primary_key: true)
end
end
end

@ -0,0 +1,9 @@
defmodule Explorer.Repo.Filecoin.Migrations.ChangeNullRoundHeightsHeightType do
use Ecto.Migration
def change do
alter table(:null_round_heights) do
modify(:height, :bigint)
end
end
end

@ -20,6 +20,7 @@ Ecto.Adapters.SQL.Sandbox.mode(Explorer.Repo.Shibarium, :auto)
Ecto.Adapters.SQL.Sandbox.mode(Explorer.Repo.Suave, :auto)
Ecto.Adapters.SQL.Sandbox.mode(Explorer.Repo.Beacon, :auto)
Ecto.Adapters.SQL.Sandbox.mode(Explorer.Repo.BridgedTokens, :auto)
Ecto.Adapters.SQL.Sandbox.mode(Explorer.Repo.Filecoin, :auto)
Mox.defmock(Explorer.ExchangeRates.Source.TestSource, for: Explorer.ExchangeRates.Source)
Mox.defmock(Explorer.Market.History.Source.Price.TestSource, for: Explorer.Market.History.Source.Price)

@ -24,6 +24,7 @@ defmodule Indexer.Block.Catchup.Fetcher do
alias Ecto.Changeset
alias Explorer.Chain
alias Explorer.Chain.NullRoundHeight
alias Explorer.Utility.MissingRangesManipulator
alias Indexer.{Block, Tracer}
alias Indexer.Block.Catchup.{Sequence, TaskSupervisor}
@ -200,7 +201,8 @@ defmodule Indexer.Block.Catchup.Fetcher do
case result do
{:ok, %{inserted: inserted, errors: errors}} ->
errors = cap_seq(sequence, errors)
valid_errors = handle_null_rounds(errors)
errors = cap_seq(sequence, valid_errors)
retry(sequence, errors)
clear_missing_ranges(range, errors)
@ -252,6 +254,20 @@ defmodule Indexer.Block.Catchup.Fetcher do
{:error, exception}
end
defp handle_null_rounds(errors) do
{null_rounds, other_errors} =
Enum.split_with(errors, fn
%{message: "requested epoch was a null round"} -> true
_ -> false
end)
null_rounds
|> Enum.map(&block_error_to_number/1)
|> NullRoundHeight.insert_heights()
other_errors
end
defp cap_seq(seq, errors) do
{not_founds, other_errors} =
Enum.split_with(errors, fn

@ -15,7 +15,7 @@ defmodule Indexer.Fetcher.TransactionAction do
alias Explorer.{Chain, Repo}
alias Explorer.Helper, as: ExplorerHelper
alias Explorer.Chain.{Block, Log, TransactionAction}
alias Explorer.Chain.{Block, BlockNumberHelper, Log, TransactionAction}
alias Indexer.Transform.{Addresses, TransactionActions}
@stage_first_block "tx_action_first_block"
@ -157,7 +157,7 @@ defmodule Indexer.Fetcher.TransactionAction do
|> Decimal.round(2)
|> Decimal.to_string()
next_block_new = block_number - 1
next_block_new = BlockNumberHelper.previous_block_number(block_number)
Logger.info(
"Block #{block_number} handled successfully. Progress: #{progress_percentage}%. Initial block range: #{first_block}..#{last_block}." <>

@ -15,6 +15,7 @@ defmodule ConfigHelper do
"rsk" -> base_repos ++ [Explorer.Repo.RSK]
"shibarium" -> base_repos ++ [Explorer.Repo.Shibarium]
"suave" -> base_repos ++ [Explorer.Repo.Suave]
"filecoin" -> base_repos ++ [Explorer.Repo.Filecoin]
_ -> base_repos
end

@ -133,6 +133,13 @@ config :explorer, Explorer.Repo.Suave,
url: ExplorerConfigHelper.get_suave_db_url(),
pool_size: 1
# Configure Filecoin database
config :explorer, Explorer.Repo.Filecoin,
database: database,
hostname: hostname,
url: System.get_env("DATABASE_URL"),
pool_size: 1
variant = Variant.get()
Code.require_file("#{variant}.exs", "apps/explorer/config/dev")

@ -101,6 +101,12 @@ config :explorer, Explorer.Repo.Suave,
pool_size: 1,
ssl: ExplorerConfigHelper.ssl_enabled?()
# Configures Filecoin database
config :explorer, Explorer.Repo.Filecoin,
url: System.get_env("DATABASE_URL"),
pool_size: 1,
ssl: ExplorerConfigHelper.ssl_enabled?()
variant = Variant.get()
Code.require_file("#{variant}.exs", "apps/explorer/config/prod")

@ -573,7 +573,9 @@
"checkproxyverification",
"NOTOK",
"sushiswap",
"zetachain"
"zetachain",
"filecoin",
"Filecoin"
],
"enableFiletypes": [
"dotenv",

Loading…
Cancel
Save