Add configurable limit for token balance fetcher init

pull/8459/head
Qwerty5Uiop 1 year ago
parent 27808373fd
commit e0941b16dc
  1. 10
      apps/explorer/lib/explorer/chain.ex
  2. 3
      apps/indexer/lib/indexer/fetcher/token_balance.ex
  3. 4
      config/runtime.exs
  4. 1
      docker-compose/envs/common-blockscout.env
  5. 3
      docker/Makefile

@ -2247,7 +2247,7 @@ defmodule Explorer.Chain do
when accumulator: term()
def stream_unfetched_token_balances(initial, reducer, limited? \\ false) when is_function(reducer, 2) do
TokenBalance.unfetched_token_balances()
|> add_fetcher_limit(limited?)
|> add_token_balances_fetcher_limit(limited?)
|> Repo.stream_reduce(initial, reducer)
end
@ -6307,6 +6307,14 @@ defmodule Explorer.Chain do
limit(query, ^fetcher_limit)
end
defp add_token_balances_fetcher_limit(query, false), do: query
defp add_token_balances_fetcher_limit(query, true) do
token_balances_fetcher_limit = Application.get_env(:indexer, :token_balances_fetcher_init_limit)
limit(query, ^token_balances_fetcher_limit)
end
def put_has_token_transfers_to_tx(query, true), do: query
def put_has_token_transfers_to_tx(query, false) do

@ -76,7 +76,8 @@ defmodule Indexer.Fetcher.TokenBalance do
token_balance
|> entry()
|> reducer.(acc)
end
end,
true
)
final

@ -422,7 +422,9 @@ config :indexer,
receipts_batch_size: ConfigHelper.parse_integer_env_var("INDEXER_RECEIPTS_BATCH_SIZE", 250),
receipts_concurrency: ConfigHelper.parse_integer_env_var("INDEXER_RECEIPTS_CONCURRENCY", 10),
hide_indexing_progress_alert: ConfigHelper.parse_bool_env_var("INDEXER_HIDE_INDEXING_PROGRESS_ALERT"),
fetcher_init_limit: ConfigHelper.parse_integer_env_var("INDEXER_FETCHER_INIT_QUERY_LIMIT", 100)
fetcher_init_limit: ConfigHelper.parse_integer_env_var("INDEXER_FETCHER_INIT_QUERY_LIMIT", 100),
token_balances_fetcher_init_limit:
ConfigHelper.parse_integer_env_var("INDEXER_TOKEN_BALANCES_FETCHER_INIT_QUERY_LIMIT", 100_000)
config :indexer, Indexer.Supervisor, enabled: !ConfigHelper.parse_bool_env_var("DISABLE_INDEXER")

@ -127,6 +127,7 @@ INDEXER_DISABLE_INTERNAL_TRANSACTIONS_FETCHER=false
# INDEXER_TX_ACTIONS_AAVE_V3_POOL_CONTRACT=
# INDEXER_REALTIME_FETCHER_MAX_GAP=
# INDEXER_FETCHER_INIT_QUERY_LIMIT=
# INDEXER_TOKEN_BALANCES_FETCHER_INIT_QUERY_LIMIT=
# INDEXER_DISABLE_WITHDRAWALS_FETCHER=
# WITHDRAWALS_FIRST_BLOCK=
# TOKEN_ID_MIGRATION_FIRST_BLOCK=

@ -582,6 +582,9 @@ endif
ifdef INDEXER_FETCHER_INIT_QUERY_LIMIT
BLOCKSCOUT_CONTAINER_PARAMS += -e 'INDEXER_FETCHER_INIT_QUERY_LIMIT=$(INDEXER_FETCHER_INIT_QUERY_LIMIT)'
endif
ifdef INDEXER_TOKEN_BALANCES_FETCHER_INIT_QUERY_LIMIT
BLOCKSCOUT_CONTAINER_PARAMS += -e 'INDEXER_TOKEN_BALANCES_FETCHER_INIT_QUERY_LIMIT=$(INDEXER_TOKEN_BALANCES_FETCHER_INIT_QUERY_LIMIT)'
endif
ifdef INDEXER_INTERNAL_TRANSACTIONS_TRACER_TYPE
BLOCKSCOUT_CONTAINER_PARAMS += -e 'INDEXER_INTERNAL_TRANSACTIONS_TRACER_TYPE=$(INDEXER_INTERNAL_TRANSACTIONS_TRACER_TYPE)'
endif

Loading…
Cancel
Save