Merge pull request #8960 from blockscout/ap-trace-block-ranges

TRACE_BLOCK_RANGES env var
pull/9049/head
Victor Baranov 11 months ago committed by GitHub
commit 7445944abf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 26
      .github/workflows/config.yml
  2. 1
      CHANGELOG.md
  3. 32
      apps/ethereum_jsonrpc/lib/ethereum_jsonrpc.ex
  4. 116
      apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/utility/ranges_helper.ex
  5. 3
      apps/explorer/lib/explorer/chain.ex
  6. 18
      apps/explorer/lib/explorer/chain/import/runner/blocks.ex
  7. 24
      apps/explorer/lib/explorer/chain/import/runner/internal_transactions.ex
  8. 23
      apps/explorer/test/explorer/chain/import/runner/internal_transactions_test.exs
  9. 39
      apps/indexer/lib/indexer/block/catchup/helper.ex
  10. 9
      apps/indexer/lib/indexer/block/catchup/missing_ranges_collector.ex
  11. 2
      apps/indexer/lib/indexer/fetcher/block_reward.ex
  12. 9
      apps/indexer/lib/indexer/fetcher/coin_balance.ex
  13. 3
      apps/indexer/lib/indexer/fetcher/internal_transaction.ex
  14. 14
      config/runtime.exs
  15. 1
      docker-compose/envs/common-blockscout.env

@ -74,7 +74,7 @@ jobs:
path: |
deps
_build
key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_30-${{ hashFiles('mix.lock') }}
key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_32-${{ hashFiles('mix.lock') }}
restore-keys: |
${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-
@ -132,7 +132,7 @@ jobs:
path: |
deps
_build
key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_30-${{ hashFiles('mix.lock') }}
key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_32-${{ hashFiles('mix.lock') }}
restore-keys: |
${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-"
@ -156,7 +156,7 @@ jobs:
path: |
deps
_build
key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_30-${{ hashFiles('mix.lock') }}
key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_32-${{ hashFiles('mix.lock') }}
restore-keys: |
${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-"
@ -185,7 +185,7 @@ jobs:
path: |
deps
_build
key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_30-${{ hashFiles('mix.lock') }}
key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_32-${{ hashFiles('mix.lock') }}
restore-keys: |
${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-"
@ -229,7 +229,7 @@ jobs:
path: |
deps
_build
key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_30-${{ hashFiles('mix.lock') }}
key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_32-${{ hashFiles('mix.lock') }}
restore-keys: |
${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-"
@ -255,7 +255,7 @@ jobs:
path: |
deps
_build
key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_30-${{ hashFiles('mix.lock') }}
key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_32-${{ hashFiles('mix.lock') }}
restore-keys: |
${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-"
@ -284,7 +284,7 @@ jobs:
path: |
deps
_build
key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_30-${{ hashFiles('mix.lock') }}
key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_32-${{ hashFiles('mix.lock') }}
restore-keys: |
${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-"
@ -332,7 +332,7 @@ jobs:
path: |
deps
_build
key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_30-${{ hashFiles('mix.lock') }}
key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_32-${{ hashFiles('mix.lock') }}
restore-keys: |
${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-"
@ -378,7 +378,7 @@ jobs:
path: |
deps
_build
key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_30-${{ hashFiles('mix.lock') }}
key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_32-${{ hashFiles('mix.lock') }}
restore-keys: |
${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-"
@ -440,7 +440,7 @@ jobs:
path: |
deps
_build
key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_30-${{ hashFiles('mix.lock') }}
key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_32-${{ hashFiles('mix.lock') }}
restore-keys: |
${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-"
@ -500,7 +500,7 @@ jobs:
path: |
deps
_build
key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_30-${{ hashFiles('mix.lock') }}
key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_32-${{ hashFiles('mix.lock') }}
restore-keys: |
${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-"
@ -571,7 +571,7 @@ jobs:
path: |
deps
_build
key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_30-${{ hashFiles('mix.lock') }}
key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_32-${{ hashFiles('mix.lock') }}
restore-keys: |
${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-"
@ -639,7 +639,7 @@ jobs:
path: |
deps
_build
key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_30-${{ hashFiles('mix.lock') }}
key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_32-${{ hashFiles('mix.lock') }}
restore-keys: |
${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-"

@ -8,6 +8,7 @@
- [#8997](https://github.com/blockscout/blockscout/pull/8997) - Isolate throttable error count by request method
- [#8975](https://github.com/blockscout/blockscout/pull/8975) - Add EIP-4844 compatibility (not full support yet)
- [#8972](https://github.com/blockscout/blockscout/pull/8972) - BENS integration
- [#8960](https://github.com/blockscout/blockscout/pull/8960) - TRACE_BLOCK_RANGES env var
- [#8957](https://github.com/blockscout/blockscout/pull/8957) - Add Tx Interpreter Service integration
### Fixes

@ -39,6 +39,7 @@ defmodule EthereumJSONRPC do
Subscription,
Transport,
Utility.EndpointAvailabilityObserver,
Utility.RangesHelper,
Variant
}
@ -205,7 +206,8 @@ defmodule EthereumJSONRPC do
filtered_params_in_range =
filtered_params
|> Enum.filter(fn
%{block_quantity: block_quantity} -> is_block_number_in_range?(block_quantity)
%{block_quantity: block_quantity} ->
block_quantity |> quantity_to_integer() |> RangesHelper.traceable_block_number?()
end)
id_to_params = id_to_params(filtered_params_in_range)
@ -244,7 +246,7 @@ defmodule EthereumJSONRPC do
@spec fetch_beneficiaries([block_number], json_rpc_named_arguments) ::
{:ok, FetchedBeneficiaries.t()} | {:error, reason :: term} | :ignore
def fetch_beneficiaries(block_numbers, json_rpc_named_arguments) when is_list(block_numbers) do
filtered_block_numbers = are_block_numbers_in_range?(block_numbers)
filtered_block_numbers = RangesHelper.filter_traceable_block_numbers(block_numbers)
Keyword.fetch!(json_rpc_named_arguments, :variant).fetch_beneficiaries(
filtered_block_numbers,
@ -349,7 +351,7 @@ defmodule EthereumJSONRPC do
Fetches internal transactions for entire blocks from variant API.
"""
def fetch_block_internal_transactions(block_numbers, json_rpc_named_arguments) when is_list(block_numbers) do
filtered_block_numbers = are_block_numbers_in_range?(block_numbers)
filtered_block_numbers = RangesHelper.filter_traceable_block_numbers(block_numbers)
Keyword.fetch!(json_rpc_named_arguments, :variant).fetch_block_internal_transactions(
filtered_block_numbers,
@ -357,16 +359,6 @@ defmodule EthereumJSONRPC do
)
end
def are_block_numbers_in_range?(block_numbers) do
min_block = Application.get_env(:indexer, :trace_first_block)
max_block = Application.get_env(:indexer, :trace_last_block)
block_numbers
|> Enum.filter(fn block_number ->
block_number >= min_block && if max_block, do: block_number <= max_block, else: true
end)
end
@doc """
Retrieves traces from variant API.
"""
@ -459,20 +451,6 @@ defmodule EthereumJSONRPC do
end
end
@spec is_block_number_in_range?(quantity) :: boolean()
defp is_block_number_in_range?(block_quantity) do
min_block = Application.get_env(:indexer, :trace_first_block)
max_block = Application.get_env(:indexer, :trace_last_block)
block_number = quantity_to_integer(block_quantity)
if !block_number ||
(block_number && block_number >= min_block && if(max_block, do: block_number <= max_block, else: true)) do
true
else
false
end
end
defp maybe_replace_url(url, _replace_url, EthereumJSONRPC.HTTP), do: url
defp maybe_replace_url(url, replace_url, _), do: EndpointAvailabilityObserver.maybe_replace_url(url, replace_url)

@ -0,0 +1,116 @@
# credo:disable-for-this-file
defmodule EthereumJSONRPC.Utility.RangesHelper do
@moduledoc """
Helper for ranges manipulations.
"""
@default_trace_block_ranges "0..latest"
@spec traceable_block_number?(integer() | nil) :: boolean()
def traceable_block_number?(block_number) do
if trace_ranges_present?() do
number_in_ranges?(block_number, get_trace_block_ranges())
else
true
end
end
@spec filter_traceable_block_numbers([integer()]) :: [integer()]
def filter_traceable_block_numbers(block_numbers) do
if trace_ranges_present?() do
trace_block_ranges = get_trace_block_ranges()
Enum.filter(block_numbers, &number_in_ranges?(&1, trace_block_ranges))
else
block_numbers
end
end
@spec trace_ranges_present? :: boolean()
def trace_ranges_present? do
Application.get_env(:indexer, :trace_block_ranges) != @default_trace_block_ranges
end
@spec get_trace_block_ranges :: [Range.t() | integer()]
def get_trace_block_ranges do
:indexer
|> Application.get_env(:trace_block_ranges)
|> parse_block_ranges()
end
@spec parse_block_ranges(binary()) :: [Range.t() | integer()]
def parse_block_ranges(block_ranges_string) do
block_ranges_string
|> String.split(",")
|> Enum.map(fn string_range ->
case String.split(string_range, "..") do
[from_string, "latest"] ->
parse_integer(from_string)
[from_string, to_string] ->
get_from_to(from_string, to_string)
_ ->
nil
end
end)
|> sanitize_ranges()
end
defp number_in_ranges?(number, ranges) do
Enum.reduce_while(ranges, false, fn
_from.._to = range, _acc -> if number in range, do: {:halt, true}, else: {:cont, false}
num_to_latest, _acc -> if number >= num_to_latest, do: {:halt, true}, else: {:cont, false}
end)
end
defp get_from_to(from_string, to_string) do
with {from, ""} <- Integer.parse(from_string),
{to, ""} <- Integer.parse(to_string) do
if from <= to, do: from..to, else: nil
else
_ -> nil
end
end
@spec sanitize_ranges([Range.t() | integer()]) :: [Range.t() | integer()]
def sanitize_ranges(ranges) do
ranges
|> Enum.reject(&is_nil/1)
|> Enum.sort_by(
fn
from.._to -> from
el -> el
end,
:asc
)
|> Enum.chunk_while(
nil,
fn
_from.._to = chunk, nil ->
{:cont, chunk}
_ch_from..ch_to = chunk, acc_from..acc_to = acc ->
if Range.disjoint?(chunk, acc),
do: {:cont, acc, chunk},
else: {:cont, acc_from..max(ch_to, acc_to)}
num, nil ->
{:halt, num}
num, acc_from.._ = acc ->
if Range.disjoint?(num..num, acc), do: {:cont, acc, num}, else: {:halt, acc_from}
_, num ->
{:halt, num}
end,
fn remainder -> {:cont, remainder, nil} end
)
end
defp parse_integer(string) do
case Integer.parse(string) do
{number, ""} -> number
_ -> nil
end
end
end

@ -33,6 +33,7 @@ defmodule Explorer.Chain do
alias Ecto.{Changeset, Multi}
alias EthereumJSONRPC.Transaction, as: EthereumJSONRPCTransaction
alias EthereumJSONRPC.Utility.RangesHelper
alias Explorer.Account.WatchlistAddress
@ -4766,7 +4767,7 @@ defmodule Explorer.Chain do
if transaction_index == 0 do
0
else
filtered_block_numbers = EthereumJSONRPC.are_block_numbers_in_range?([block_number])
filtered_block_numbers = RangesHelper.filter_traceable_block_numbers([block_number])
{:ok, traces} = fetch_block_internal_transactions(filtered_block_numbers, json_rpc_named_arguments)
sorted_traces =

@ -9,6 +9,8 @@ defmodule Explorer.Chain.Import.Runner.Blocks do
alias Ecto.{Changeset, Multi, Repo}
alias EthereumJSONRPC.Utility.RangesHelper
alias Explorer.Chain.{
Address,
Block,
@ -62,7 +64,7 @@ defmodule Explorer.Chain.Import.Runner.Blocks do
items_for_pending_ops =
changes_list
|> filter_by_height_range(&is_block_in_range?(&1.number))
|> filter_by_height_range(&RangesHelper.traceable_block_number?(&1.number))
|> Enum.filter(& &1.consensus)
|> Enum.map(&{&1.number, &1.hash})
@ -72,7 +74,8 @@ defmodule Explorer.Chain.Import.Runner.Blocks do
run_func = fn repo ->
{:ok, nonconsensus_items} = lose_consensus(repo, hashes, consensus_block_numbers, changes_list, insert_options)
{:ok, filter_by_height_range(nonconsensus_items, fn {number, _hash} -> is_block_in_range?(number) end)}
{:ok,
filter_by_height_range(nonconsensus_items, fn {number, _hash} -> RangesHelper.traceable_block_number?(number) end)}
end
multi
@ -214,12 +217,6 @@ defmodule Explorer.Chain.Import.Runner.Blocks do
@impl Runner
def timeout, do: @timeout
defp is_block_in_range?(number) do
minimal_block_height = Application.get_env(:indexer, :trace_first_block)
maximal_block_height = Application.get_env(:indexer, :trace_last_block)
number >= minimal_block_height && if(maximal_block_height, do: number <= maximal_block_height, else: true)
end
defp fork_transactions(%{
repo: repo,
timeout: timeout,
@ -890,10 +887,7 @@ defmodule Explorer.Chain.Import.Runner.Blocks do
end
defp filter_by_height_range(blocks, filter_func) do
minimal_block_height = Application.get_env(:indexer, :trace_first_block)
maximal_block_height = Application.get_env(:indexer, :trace_last_block)
if minimal_block_height > 0 || maximal_block_height do
if RangesHelper.trace_ranges_present?() do
Enum.filter(blocks, &filter_func.(&1))
else
blocks

@ -8,6 +8,7 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactions do
alias Ecto.Adapters.SQL
alias Ecto.{Changeset, Multi, Repo}
alias EthereumJSONRPC.Utility.RangesHelper
alias Explorer.Chain.{Block, Hash, Import, InternalTransaction, PendingBlockOperation, Transaction}
alias Explorer.Chain.Events.Publisher
alias Explorer.Chain.Import.Runner
@ -15,7 +16,7 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactions do
alias Explorer.Repo, as: ExplorerRepo
alias Explorer.Utility.MissingRangesManipulator
import Ecto.Query, only: [from: 2, where: 3]
import Ecto.Query
@behaviour Runner
@ -691,23 +692,17 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactions do
end
defp remove_consensus_of_invalid_blocks(repo, invalid_block_numbers) do
minimal_block = Application.get_env(:indexer, :trace_first_block)
maximal_block = Application.get_env(:indexer, :trace_last_block)
if Enum.count(invalid_block_numbers) > 0 do
update_query =
from(
block in Block,
where: block.number in ^invalid_block_numbers and block.consensus == true,
where: block.number > ^minimal_block,
where: ^traceable_blocks_dynamic_query(),
select: block.hash,
# ShareLocks order already enforced by `acquire_blocks` (see docs: sharelocks.md)
update: [set: [consensus: false]]
)
update_query =
if maximal_block, do: update_query |> where([block], block.number < ^maximal_block), else: update_query
try do
{_num, result} = repo.update_all(update_query, [])
@ -754,4 +749,17 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactions do
{:error, %{exception: postgrex_error, pending_hashes: valid_block_hashes}}
end
end
defp traceable_blocks_dynamic_query do
if RangesHelper.trace_ranges_present?() do
block_ranges = RangesHelper.get_trace_block_ranges()
Enum.reduce(block_ranges, dynamic([_], false), fn
_from.._to = range, acc -> dynamic([block], ^acc or block.number in ^range)
num_to_latest, acc -> dynamic([block], ^acc or block.number >= ^num_to_latest)
end)
else
dynamic([_], true)
end
end
end

@ -306,6 +306,29 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactionsTest do
assert PendingBlockOperation |> Repo.get(full_block.hash) |> is_nil()
end
test "does not remove consensus from non-traceable blocks" do
original_config = Application.get_env(:indexer, :trace_block_ranges)
full_block = insert(:block)
transaction_a = insert(:transaction) |> with_block(full_block)
transaction_b = insert(:transaction) |> with_block(full_block)
Application.put_env(:indexer, :trace_block_ranges, "#{full_block.number + 1}..latest")
insert(:pending_block_operation, block_hash: full_block.hash, block_number: full_block.number)
transaction_a_changes = make_internal_transaction_changes(transaction_a, 0, nil)
assert {:ok, _} = run_internal_transactions([transaction_a_changes])
assert from(i in InternalTransaction, where: i.transaction_hash == ^transaction_a.hash) |> Repo.one() |> is_nil()
assert from(i in InternalTransaction, where: i.transaction_hash == ^transaction_b.hash) |> Repo.one() |> is_nil()
assert %{consensus: true} = Repo.get(Block, full_block.hash)
on_exit(fn -> Application.put_env(:indexer, :trace_block_ranges, original_config) end)
end
test "successfully imports internal transaction with stop type" do
block = insert(:block)
transaction = insert(:transaction) |> with_block(block, status: :ok)

@ -1,39 +0,0 @@
defmodule Indexer.Block.Catchup.Helper do
@moduledoc """
Catchup helper functions
"""
def sanitize_ranges(ranges) do
ranges
|> Enum.filter(&(not is_nil(&1)))
|> Enum.sort_by(
fn
from.._to -> from
el -> el
end,
:asc
)
|> Enum.chunk_while(
nil,
fn
_from.._to = chunk, nil ->
{:cont, chunk}
_ch_from..ch_to = chunk, acc_from..acc_to = acc ->
if Range.disjoint?(chunk, acc),
do: {:cont, acc, chunk},
else: {:cont, acc_from..max(ch_to, acc_to)}
num, nil ->
{:halt, num}
num, acc_from.._ = acc ->
if Range.disjoint?(num..num, acc), do: {:cont, acc, num}, else: {:halt, acc_from}
_, num ->
{:halt, num}
end,
fn reminder -> {:cont, reminder, nil} end
)
end
end

@ -5,11 +5,10 @@ defmodule Indexer.Block.Catchup.MissingRangesCollector do
use GenServer
alias Explorer.{Chain, Repo}
alias EthereumJSONRPC.Utility.RangesHelper
alias Explorer.{Chain, Helper, Repo}
alias Explorer.Chain.Cache.BlockNumber
alias Explorer.Helper, as: ExplorerHelper
alias Explorer.Utility.{MissingBlockRange, MissingRangesManipulator}
alias Indexer.Block.Catchup.Helper
@default_missing_ranges_batch_size 100_000
@future_check_interval Application.compile_env(:indexer, __MODULE__)[:future_check_interval]
@ -233,7 +232,7 @@ defmodule Indexer.Block.Catchup.MissingRangesCollector do
|> Enum.map(fn string_range ->
case String.split(string_range, "..") do
[from_string, "latest"] ->
ExplorerHelper.parse_integer(from_string)
Helper.parse_integer(from_string)
[from_string, to_string] ->
get_from_to(from_string, to_string)
@ -242,7 +241,7 @@ defmodule Indexer.Block.Catchup.MissingRangesCollector do
nil
end
end)
|> Helper.sanitize_ranges()
|> RangesHelper.sanitize_ranges()
case List.last(ranges) do
_from.._to ->

@ -21,7 +21,7 @@ defmodule Indexer.Fetcher.BlockReward do
alias Indexer.{BufferedTask, Tracer}
alias Indexer.Fetcher.BlockReward.Supervisor, as: BlockRewardSupervisor
alias Indexer.Fetcher.CoinBalance
alias Indexer.Transform.{AddressCoinBalances, AddressCoinBalancesDaily, Addresses}
alias Indexer.Transform.{AddressCoinBalances, Addresses}
@behaviour BufferedTask

@ -11,7 +11,7 @@ defmodule Indexer.Fetcher.CoinBalance do
import EthereumJSONRPC, only: [integer_to_quantity: 1, quantity_to_integer: 1]
alias EthereumJSONRPC.{Blocks, FetchedBalances}
alias EthereumJSONRPC.{Blocks, FetchedBalances, Utility.RangesHelper}
alias Explorer.Chain
alias Explorer.Chain.{Block, Hash}
alias Explorer.Chain.Cache.Accounts
@ -83,13 +83,8 @@ defmodule Indexer.Fetcher.CoinBalance do
# `{address, block}`, so take unique params only
unique_entries = Enum.uniq(entries)
min_block = Application.get_env(:indexer, :trace_first_block)
max_block = Application.get_env(:indexer, :trace_last_block)
unique_filtered_entries =
Enum.filter(unique_entries, fn {_hash, block_number} ->
block_number >= min_block && if max_block, do: block_number <= max_block, else: true
end)
Enum.filter(unique_entries, fn {_hash, block_number} -> RangesHelper.traceable_block_number?(block_number) end)
unique_entry_count = Enum.count(unique_filtered_entries)
Logger.metadata(count: unique_entry_count)

@ -12,6 +12,7 @@ defmodule Indexer.Fetcher.InternalTransaction do
import Indexer.Block.Fetcher, only: [async_import_coin_balances: 2]
alias EthereumJSONRPC.Utility.RangesHelper
alias Explorer.Chain
alias Explorer.Chain.Block
alias Explorer.Chain.Cache.{Accounts, Blocks}
@ -100,7 +101,7 @@ defmodule Indexer.Fetcher.InternalTransaction do
filtered_unique_numbers =
unique_numbers
|> EthereumJSONRPC.are_block_numbers_in_range?()
|> RangesHelper.filter_traceable_block_numbers()
|> drop_genesis(json_rpc_named_arguments)
filtered_unique_numbers_count = Enum.count(filtered_unique_numbers)

@ -459,14 +459,24 @@ config :explorer, Explorer.MicroserviceInterfaces.BENS,
### Indexer ###
###############
trace_first_block = ConfigHelper.parse_integer_env_var("TRACE_FIRST_BLOCK", 0)
trace_last_block = ConfigHelper.parse_integer_or_nil_env_var("TRACE_LAST_BLOCK")
trace_block_ranges =
case ConfigHelper.safe_get_env("TRACE_BLOCK_RANGES", nil) do
"" -> "#{trace_first_block}..#{trace_last_block || "latest"}"
ranges -> ranges
end
config :indexer,
block_transformer: ConfigHelper.block_transformer(),
metadata_updater_milliseconds_interval: ConfigHelper.parse_time_env_var("TOKEN_METADATA_UPDATE_INTERVAL", "48h"),
block_ranges: System.get_env("BLOCK_RANGES"),
first_block: ConfigHelper.parse_integer_env_var("FIRST_BLOCK", 0),
last_block: ConfigHelper.parse_integer_or_nil_env_var("LAST_BLOCK"),
trace_first_block: ConfigHelper.parse_integer_env_var("TRACE_FIRST_BLOCK", 0),
trace_last_block: ConfigHelper.parse_integer_or_nil_env_var("TRACE_LAST_BLOCK"),
trace_block_ranges: trace_block_ranges,
trace_first_block: trace_first_block,
trace_last_block: trace_last_block,
fetch_rewards_way: System.get_env("FETCH_REWARDS_WAY", "trace_block"),
memory_limit: ConfigHelper.indexer_memory_limit(),
receipts_batch_size: ConfigHelper.parse_integer_env_var("INDEXER_RECEIPTS_BATCH_SIZE", 250),

@ -63,6 +63,7 @@ BLOCK_TRANSFORMER=base
# BLOCK_RANGES=
# FIRST_BLOCK=
# LAST_BLOCK=
# TRACE_BLOCK_RANGES=
# TRACE_FIRST_BLOCK=
# TRACE_LAST_BLOCK=
# FOOTER_CHAT_LINK=

Loading…
Cancel
Save