Disable logging debug_count at regular intervals

Fixes #422

BlockFetcher's debug_count support used the slow `Explorer.Chain.*_count`
functions.  We want those counts to be completely accurate when needed, so we
can't move to the `Explorer.Chain.*_estimated_count`.  Since, the
current, accurate counts are slowing down the indexing by causing a
timeout to have to be handled, completely remove the debug count support
and have developers call the `Explorer.Chain.*_count` when they need
them to monitor progress.
pull/437/head
Luke Imhoff 6 years ago
parent 0d8cc39bd2
commit 3432fe4c51
  1. 28
      apps/indexer/lib/indexer/block_fetcher.ex
  2. 84
      apps/indexer/test/indexer/block_fetcher_test.exs

@ -11,7 +11,7 @@ defmodule Indexer.BlockFetcher do
alias EthereumJSONRPC alias EthereumJSONRPC
alias Explorer.Chain alias Explorer.Chain
alias Indexer.{AddressBalanceFetcher, AddressExtraction, BufferedTask, InternalTransactionFetcher, Sequence} alias Indexer.{AddressBalanceFetcher, AddressExtraction, InternalTransactionFetcher, Sequence}
# dialyzer thinks that Logger.debug functions always have no_local_return # dialyzer thinks that Logger.debug functions always have no_local_return
@dialyzer {:nowarn_function, import_range: 3} @dialyzer {:nowarn_function, import_range: 3}
@ -66,8 +66,6 @@ defmodule Indexer.BlockFetcher do
|> Application.get_all_env() |> Application.get_all_env()
|> Keyword.merge(opts) |> Keyword.merge(opts)
:timer.send_interval(15_000, self(), :debug_count)
state = %{ state = %{
json_rpc_named_arguments: Keyword.fetch!(opts, :json_rpc_named_arguments), json_rpc_named_arguments: Keyword.fetch!(opts, :json_rpc_named_arguments),
genesis_task: nil, genesis_task: nil,
@ -121,30 +119,6 @@ defmodule Indexer.BlockFetcher do
{:noreply, schedule_next_catchup_index(%{state | genesis_task: nil})} {:noreply, schedule_next_catchup_index(%{state | genesis_task: nil})}
end end
def handle_info(:debug_count, %{} = state) do
debug(fn ->
"""
================================
persisted counts
================================
addresses: #{Chain.address_count()}
blocks: #{Chain.block_count()}
internal transactions: #{Chain.internal_transaction_count()}
logs: #{Chain.log_count()}
addresses: #{Chain.address_count()}
================================
deferred fetches
================================
address balances: #{inspect(BufferedTask.debug_count(AddressBalanceFetcher))}
internal transactions: #{inspect(BufferedTask.debug_count(InternalTransactionFetcher))}
"""
end)
{:noreply, state}
end
defp cap_seq(seq, next, range) do defp cap_seq(seq, next, range) do
case next do case next do
:more -> :more ->

@ -3,7 +3,6 @@ defmodule Indexer.BlockFetcherTest do
use EthereumJSONRPC.Case, async: false use EthereumJSONRPC.Case, async: false
use Explorer.DataCase use Explorer.DataCase
import ExUnit.CaptureLog
import Mox import Mox
import EthereumJSONRPC, only: [integer_to_quantity: 1] import EthereumJSONRPC, only: [integer_to_quantity: 1]
import EthereumJSONRPC.Case import EthereumJSONRPC.Case
@ -241,64 +240,6 @@ defmodule Indexer.BlockFetcherTest do
end end
end end
describe "handle_info(:debug_count, state)" do
setup :state
setup do
block = insert(:block)
Enum.map(0..2, fn _ ->
transaction =
:transaction
|> insert()
|> with_block(block)
insert(:log, transaction: transaction)
insert(:internal_transaction, transaction: transaction, index: 0)
end)
:ok
end
@tag :capture_log
@heading "persisted counts"
test "without debug_logs", %{json_rpc_named_arguments: json_rpc_named_arguments, state: state} do
start_supervised!({Task.Supervisor, name: Indexer.TaskSupervisor})
AddressBalanceFetcherCase.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments)
InternalTransactionFetcherCase.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments)
wait_for_tasks(InternalTransactionFetcher)
wait_for_tasks(AddressBalanceFetcher)
refute capture_log_at_level(:debug, fn ->
Indexer.disable_debug_logs()
BlockFetcher.handle_info(:debug_count, state)
end) =~ @heading
end
@tag :capture_log
test "with debug_logs", %{json_rpc_named_arguments: json_rpc_named_arguments, state: state} do
start_supervised!({Task.Supervisor, name: Indexer.TaskSupervisor})
AddressBalanceFetcherCase.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments)
InternalTransactionFetcherCase.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments)
wait_for_tasks(InternalTransactionFetcher)
wait_for_tasks(AddressBalanceFetcher)
log =
capture_log_at_level(:debug, fn ->
Indexer.enable_debug_logs()
BlockFetcher.handle_info(:debug_count, state)
end)
assert log =~ @heading
assert log =~ "blocks: 1"
assert log =~ "internal transactions: 3"
assert log =~ "logs: 3"
assert log =~ "addresses: 16"
end
end
describe "import_range/3" do describe "import_range/3" do
setup :state setup :state
@ -829,31 +770,6 @@ defmodule Indexer.BlockFetcherTest do
end end
end end
defp capture_log_at_level(level, block) do
logger_level_transaction(fn ->
Logger.configure(level: level)
capture_log(fn ->
block.()
Process.sleep(10)
end)
end)
end
defp logger_level_transaction(block) do
level_before = Logger.level()
on_exit(fn ->
Logger.configure(level: level_before)
end)
return = block.()
Logger.configure(level: level_before)
return
end
defp state(%{json_rpc_named_arguments: json_rpc_named_arguments}) do defp state(%{json_rpc_named_arguments: json_rpc_named_arguments}) do
{:ok, state} = BlockFetcher.init(json_rpc_named_arguments: json_rpc_named_arguments) {:ok, state} = BlockFetcher.init(json_rpc_named_arguments: json_rpc_named_arguments)

Loading…
Cancel
Save