Merge pull request #2718 from poanetwork/vb-addresses-counter

Include all addresses taking part in transactions in wallets' addresses counter
pull/2731/head
Victor Baranov 5 years ago committed by GitHub
commit 1f90f7eed1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 1
      CHANGELOG.md
  2. 2
      apps/block_scout_web/lib/block_scout_web/controllers/address_controller.ex
  3. 2
      apps/block_scout_web/lib/block_scout_web/controllers/chain_controller.ex
  4. 2
      apps/block_scout_web/lib/block_scout_web/notifier.ex
  5. 16
      apps/block_scout_web/test/block_scout_web/channels/address_channel_test.exs
  6. 10
      apps/block_scout_web/test/block_scout_web/controllers/address_controller_test.exs
  7. 4
      apps/block_scout_web/test/block_scout_web/controllers/api/rpc/address_controller_test.exs
  8. 4
      apps/block_scout_web/test/block_scout_web/controllers/api/rpc/eth_controller_test.exs
  9. 8
      apps/block_scout_web/test/block_scout_web/controllers/chain_controller_test.exs
  10. 6
      apps/block_scout_web/test/block_scout_web/features/viewing_addresses_test.exs
  11. 6
      apps/block_scout_web/test/block_scout_web/features/viewing_app_test.exs
  12. 34
      apps/block_scout_web/test/block_scout_web/features/viewing_chain_test.exs
  13. 5
      apps/explorer/config/config.exs
  14. 2
      apps/explorer/config/test.exs
  15. 1
      apps/explorer/lib/explorer/application.ex
  16. 23
      apps/explorer/lib/explorer/chain.ex
  17. 10
      apps/explorer/lib/explorer/chain/address.ex
  18. 125
      apps/explorer/lib/explorer/counters/addresses_counter.ex
  19. 17
      apps/explorer/test/explorer/chain_test.exs
  20. 16
      apps/explorer/test/explorer/counters/addresses_counter_test.exs

@ -11,6 +11,7 @@
- [#2663](https://github.com/poanetwork/blockscout/pull/2663) - Fetch address counters in parallel
### Fixes
- [#2718](https://github.com/poanetwork/blockscout/pull/2718) - Include all addresses taking part in transactions in wallets' addresses counter
- [#2709](https://github.com/poanetwork/blockscout/pull/2709) - Fix stuck label and value for uncle block height
- [#2707](https://github.com/poanetwork/blockscout/pull/2707) - fix for dashboard banner chart legend items
- [#2706](https://github.com/poanetwork/blockscout/pull/2706) - fix empty total_supply in coin gecko response

@ -61,7 +61,7 @@ defmodule BlockScoutWeb.AddressController do
def index(conn, _params) do
render(conn, "index.html",
current_path: current_path(conn),
address_count: Chain.count_addresses_with_balance_from_cache()
address_count: Chain.count_addresses_from_cache()
)
end

@ -28,7 +28,7 @@ defmodule BlockScoutWeb.ChainController do
render(
conn,
"show.html",
address_count: Chain.count_addresses_with_balance_from_cache(),
address_count: Chain.count_addresses_from_cache(),
average_block_time: AverageBlockTime.average_block_time(),
exchange_rate: exchange_rate,
chart_data_path: market_history_chart_path(conn, :show),

@ -14,7 +14,7 @@ defmodule BlockScoutWeb.Notifier do
alias Phoenix.View
def handle_event({:chain_event, :addresses, type, addresses}) when type in [:realtime, :on_demand] do
Endpoint.broadcast("addresses:new_address", "count", %{count: Chain.count_addresses_with_balance_from_cache()})
Endpoint.broadcast("addresses:new_address", "count", %{count: Chain.count_addresses_from_cache()})
addresses
|> Stream.reject(fn %Address{fetched_coin_balance: fetched_coin_balance} -> is_nil(fetched_coin_balance) end)

@ -1,11 +1,11 @@
defmodule BlockScoutWeb.AddressChannelTest do
use BlockScoutWeb.ChannelCase,
# ETS tables are shared in `Explorer.Counters.AddressesWithBalanceCounter`
# ETS tables are shared in `Explorer.Counters.AddressesCounter`
async: false
alias BlockScoutWeb.UserSocket
alias BlockScoutWeb.Notifier
alias Explorer.Counters.AddressesWithBalanceCounter
alias Explorer.Counters.AddressesCounter
test "subscribed user is notified of new_address count event" do
topic = "addresses:new_address"
@ -13,8 +13,8 @@ defmodule BlockScoutWeb.AddressChannelTest do
address = insert(:address)
start_supervised!(AddressesWithBalanceCounter)
AddressesWithBalanceCounter.consolidate()
start_supervised!(AddressesCounter)
AddressesCounter.consolidate()
Notifier.handle_event({:chain_event, :addresses, :realtime, [address]})
@ -55,8 +55,8 @@ defmodule BlockScoutWeb.AddressChannelTest do
test "notified of balance_update for matching address", %{address: address, topic: topic} do
address_with_balance = %{address | fetched_coin_balance: 1}
start_supervised!(AddressesWithBalanceCounter)
AddressesWithBalanceCounter.consolidate()
start_supervised!(AddressesCounter)
AddressesCounter.consolidate()
Notifier.handle_event({:chain_event, :addresses, :realtime, [address_with_balance]})
@ -67,8 +67,8 @@ defmodule BlockScoutWeb.AddressChannelTest do
end
test "not notified of balance_update if fetched_coin_balance is nil", %{address: address} do
start_supervised!(AddressesWithBalanceCounter)
AddressesWithBalanceCounter.consolidate()
start_supervised!(AddressesCounter)
AddressesCounter.consolidate()
Notifier.handle_event({:chain_event, :addresses, :realtime, [address]})

@ -3,7 +3,7 @@ defmodule BlockScoutWeb.AddressControllerTest do
# ETS tables are shared in `Explorer.Counters.*`
async: false
alias Explorer.Counters.AddressesWithBalanceCounter
alias Explorer.Counters.AddressesCounter
describe "GET index/2" do
test "returns top addresses", %{conn: conn} do
@ -12,8 +12,8 @@ defmodule BlockScoutWeb.AddressControllerTest do
|> Enum.map(&insert(:address, fetched_coin_balance: &1))
|> Enum.map(& &1.hash)
start_supervised!(AddressesWithBalanceCounter)
AddressesWithBalanceCounter.consolidate()
start_supervised!(AddressesCounter)
AddressesCounter.consolidate()
conn = get(conn, address_path(conn, :index, %{type: "JSON"}))
{:ok, %{"items" => items}} = Poison.decode(conn.resp_body)
@ -25,8 +25,8 @@ defmodule BlockScoutWeb.AddressControllerTest do
address = insert(:address, fetched_coin_balance: 1)
insert(:address_name, address: address, primary: true, name: "POA Wallet")
start_supervised!(AddressesWithBalanceCounter)
AddressesWithBalanceCounter.consolidate()
start_supervised!(AddressesCounter)
AddressesCounter.consolidate()
conn = get(conn, address_path(conn, :index, %{type: "JSON"}))

@ -6,7 +6,7 @@ defmodule BlockScoutWeb.API.RPC.AddressControllerTest do
alias BlockScoutWeb.API.RPC.AddressController
alias Explorer.Chain
alias Explorer.Chain.{Events.Subscriber, Transaction, Wei}
alias Explorer.Counters.{AddressesWithBalanceCounter, AverageBlockTime}
alias Explorer.Counters.{AddressesCounter, AverageBlockTime}
alias Indexer.Fetcher.CoinBalanceOnDemand
alias Explorer.Repo
@ -22,7 +22,7 @@ defmodule BlockScoutWeb.API.RPC.AddressControllerTest do
start_supervised!({Task.Supervisor, name: Indexer.TaskSupervisor})
start_supervised!(AverageBlockTime)
start_supervised!({CoinBalanceOnDemand, [mocked_json_rpc_named_arguments, [name: CoinBalanceOnDemand]]})
start_supervised!(AddressesWithBalanceCounter)
start_supervised!(AddressesCounter)
Application.put_env(:explorer, AverageBlockTime, enabled: true)

@ -1,7 +1,7 @@
defmodule BlockScoutWeb.API.RPC.EthControllerTest do
use BlockScoutWeb.ConnCase, async: false
alias Explorer.Counters.{AddressesWithBalanceCounter, AverageBlockTime}
alias Explorer.Counters.{AddressesCounter, AverageBlockTime}
alias Explorer.Repo
alias Indexer.Fetcher.CoinBalanceOnDemand
@ -14,7 +14,7 @@ defmodule BlockScoutWeb.API.RPC.EthControllerTest do
start_supervised!({Task.Supervisor, name: Indexer.TaskSupervisor})
start_supervised!(AverageBlockTime)
start_supervised!({CoinBalanceOnDemand, [mocked_json_rpc_named_arguments, [name: CoinBalanceOnDemand]]})
start_supervised!(AddressesWithBalanceCounter)
start_supervised!(AddressesCounter)
Application.put_env(:explorer, AverageBlockTime, enabled: true)

@ -1,18 +1,18 @@
defmodule BlockScoutWeb.ChainControllerTest do
use BlockScoutWeb.ConnCase,
# ETS table is shared in `Explorer.Counters.AddressesWithBalanceCounter`
# ETS table is shared in `Explorer.Counters.AddressesCounter`
async: false
import BlockScoutWeb.WebRouter.Helpers, only: [chain_path: 2, block_path: 3, transaction_path: 3, address_path: 3]
alias Explorer.Chain.Block
alias Explorer.Counters.AddressesWithBalanceCounter
alias Explorer.Counters.AddressesCounter
setup do
Supervisor.terminate_child(Explorer.Supervisor, Explorer.Chain.Cache.Blocks.child_id())
Supervisor.restart_child(Explorer.Supervisor, Explorer.Chain.Cache.Blocks.child_id())
start_supervised!(AddressesWithBalanceCounter)
AddressesWithBalanceCounter.consolidate()
start_supervised!(AddressesCounter)
AddressesCounter.consolidate()
:ok
end

@ -3,7 +3,7 @@ defmodule BlockScoutWeb.ViewingAddressesTest do
# Because ETS tables is shared for `Explorer.Counters.*`
async: false
alias Explorer.Counters.AddressesWithBalanceCounter
alias Explorer.Counters.AddressesCounter
alias BlockScoutWeb.{AddressPage, AddressView, Notifier}
setup do
@ -58,8 +58,8 @@ defmodule BlockScoutWeb.ViewingAddressesTest do
[first_address | _] = addresses
[last_address | _] = Enum.reverse(addresses)
start_supervised!(AddressesWithBalanceCounter)
AddressesWithBalanceCounter.consolidate()
start_supervised!(AddressesCounter)
AddressesCounter.consolidate()
session
|> AddressPage.visit_page()

@ -5,11 +5,11 @@ defmodule BlockScoutWeb.ViewingAppTest do
alias BlockScoutWeb.AppPage
alias BlockScoutWeb.Counters.BlocksIndexedCounter
alias Explorer.Counters.AddressesWithBalanceCounter
alias Explorer.Counters.AddressesCounter
setup do
start_supervised!(AddressesWithBalanceCounter)
AddressesWithBalanceCounter.consolidate()
start_supervised!(AddressesCounter)
AddressesCounter.consolidate()
:ok
end

@ -7,7 +7,7 @@ defmodule BlockScoutWeb.ViewingChainTest do
alias BlockScoutWeb.{AddressPage, BlockPage, ChainPage, TransactionPage}
alias Explorer.Chain.Block
alias Explorer.Counters.AddressesWithBalanceCounter
alias Explorer.Counters.AddressesCounter
setup do
Supervisor.terminate_child(Explorer.Supervisor, Explorer.Chain.Cache.Blocks.child_id())
@ -35,8 +35,8 @@ defmodule BlockScoutWeb.ViewingChainTest do
test "search for address", %{session: session} do
address = insert(:address)
start_supervised!(AddressesWithBalanceCounter)
AddressesWithBalanceCounter.consolidate()
start_supervised!(AddressesCounter)
AddressesCounter.consolidate()
session
|> ChainPage.visit_page()
@ -49,8 +49,8 @@ defmodule BlockScoutWeb.ViewingChainTest do
test "search for blocks from chain page", %{session: session} do
block = insert(:block, number: 6)
start_supervised!(AddressesWithBalanceCounter)
AddressesWithBalanceCounter.consolidate()
start_supervised!(AddressesCounter)
AddressesCounter.consolidate()
session
|> ChainPage.visit_page()
@ -59,8 +59,8 @@ defmodule BlockScoutWeb.ViewingChainTest do
end
test "blocks list", %{session: session} do
start_supervised!(AddressesWithBalanceCounter)
AddressesWithBalanceCounter.consolidate()
start_supervised!(AddressesCounter)
AddressesCounter.consolidate()
session
|> ChainPage.visit_page()
@ -70,8 +70,8 @@ defmodule BlockScoutWeb.ViewingChainTest do
test "inserts place holder blocks on render for out of order blocks", %{session: session} do
insert(:block, number: 409)
start_supervised!(AddressesWithBalanceCounter)
AddressesWithBalanceCounter.consolidate()
start_supervised!(AddressesCounter)
AddressesCounter.consolidate()
session
|> ChainPage.visit_page()
@ -84,8 +84,8 @@ defmodule BlockScoutWeb.ViewingChainTest do
test "search for transactions", %{session: session} do
transaction = insert(:transaction)
start_supervised!(AddressesWithBalanceCounter)
AddressesWithBalanceCounter.consolidate()
start_supervised!(AddressesCounter)
AddressesCounter.consolidate()
session
|> ChainPage.visit_page()
@ -94,8 +94,8 @@ defmodule BlockScoutWeb.ViewingChainTest do
end
test "transactions list", %{session: session} do
start_supervised!(AddressesWithBalanceCounter)
AddressesWithBalanceCounter.consolidate()
start_supervised!(AddressesCounter)
AddressesCounter.consolidate()
session
|> ChainPage.visit_page()
@ -111,8 +111,8 @@ defmodule BlockScoutWeb.ViewingChainTest do
|> with_contract_creation(contract_address)
|> with_block(block)
start_supervised!(AddressesWithBalanceCounter)
AddressesWithBalanceCounter.consolidate()
start_supervised!(AddressesCounter)
AddressesCounter.consolidate()
session
|> ChainPage.visit_page()
@ -138,8 +138,8 @@ defmodule BlockScoutWeb.ViewingChainTest do
token_contract_address: contract_token_address
)
start_supervised!(AddressesWithBalanceCounter)
AddressesWithBalanceCounter.consolidate()
start_supervised!(AddressesCounter)
AddressesCounter.consolidate()
session
|> ChainPage.visit_page()

@ -47,6 +47,11 @@ balances_update_interval =
end
config :explorer, Explorer.Counters.AddressesWithBalanceCounter,
enabled: false,
enable_consolidation: true,
update_interval_in_seconds: balances_update_interval || 30 * 60
config :explorer, Explorer.Counters.AddressesCounter,
enabled: true,
enable_consolidation: true,
update_interval_in_seconds: balances_update_interval || 30 * 60

@ -21,6 +21,8 @@ config :explorer, Explorer.Counters.AverageBlockTime, enabled: false
config :explorer, Explorer.Counters.AddressesWithBalanceCounter, enabled: false, enable_consolidation: false
config :explorer, Explorer.Counters.AddressesCounter, enabled: false, enable_consolidation: false
config :explorer, Explorer.Market.History.Cataloger, enabled: false
config :explorer, Explorer.Tracer, disabled?: false

@ -68,6 +68,7 @@ defmodule Explorer.Application do
configure(Explorer.KnownTokens),
configure(Explorer.Market.History.Cataloger),
configure(Explorer.Counters.AddressesWithBalanceCounter),
configure(Explorer.Counters.AddressesCounter),
configure(Explorer.Counters.AverageBlockTime),
configure(Explorer.Validator.MetadataProcessor),
configure(Explorer.Staking.EpochCounter)

@ -58,7 +58,7 @@ defmodule Explorer.Chain do
}
alias Explorer.Chain.Import.Runner
alias Explorer.Counters.AddressesWithBalanceCounter
alias Explorer.Counters.{AddressesCounter, AddressesWithBalanceCounter}
alias Explorer.Market.MarketHistoryCache
alias Explorer.{PagingOptions, Repo}
@ -118,6 +118,14 @@ defmodule Explorer.Chain do
AddressesWithBalanceCounter.fetch()
end
@doc """
Gets from the cache the count of all `t:Explorer.Chain.Address.t/0`'s
"""
@spec count_addresses_from_cache :: non_neg_integer()
def count_addresses_from_cache do
AddressesCounter.fetch()
end
@doc """
Counts the number of addresses with fetched coin balance > 0.
@ -131,6 +139,19 @@ defmodule Explorer.Chain do
)
end
@doc """
Counts the number of all addresses.
This function should be used with caution. In larger databases, it may take a
while to have the return back.
"""
def count_addresses do
Repo.one(
Address.count(),
timeout: :infinity
)
end
@doc """
`t:Explorer.Chain.InternalTransaction/0`s from the address with the given `hash`.

@ -237,6 +237,16 @@ defmodule Explorer.Chain.Address do
)
end
@doc """
Counts all the addresses.
"""
def count do
from(
a in Address,
select: fragment("COUNT(*)")
)
end
defimpl String.Chars do
@doc """
Uses `hash` as string representation, formatting it according to the eip-55 specification

@ -0,0 +1,125 @@
defmodule Explorer.Counters.AddressesCounter do
@moduledoc """
Caches the number of all addresses.
It loads the count asynchronously and in a time interval of 30 minutes.
"""
use GenServer
alias Explorer.Chain
@table :addresses_counter
@cache_key "addresses"
def table_name do
@table
end
def cache_key do
@cache_key
end
# It is undesirable to automatically start the consolidation in all environments.
# Consider the test environment: if the consolidation initiates but does not
# finish before a test ends, that test will fail. This way, hundreds of
# tests were failing before disabling the consolidation and the scheduler in
# the test env.
config = Application.get_env(:explorer, Explorer.Counters.AddressesCounter)
@enable_consolidation Keyword.get(config, :enable_consolidation)
@update_interval_in_seconds Keyword.get(config, :update_interval_in_seconds)
@doc """
Starts a process to periodically update the counter of the token holders.
"""
@spec start_link(term()) :: GenServer.on_start()
def start_link(_) do
GenServer.start_link(__MODULE__, :ok, name: __MODULE__)
end
@impl true
def init(_args) do
create_table()
{:ok, %{consolidate?: enable_consolidation?()}, {:continue, :ok}}
end
def create_table do
opts = [
:set,
:named_table,
:public,
read_concurrency: true
]
:ets.new(table_name(), opts)
end
defp schedule_next_consolidation do
Process.send_after(self(), :consolidate, :timer.seconds(@update_interval_in_seconds))
end
@doc """
Inserts new items into the `:ets` table.
"""
def insert_counter({key, info}) do
:ets.insert(table_name(), {key, info})
end
@impl true
def handle_continue(:ok, %{consolidate?: true} = state) do
consolidate()
schedule_next_consolidation()
{:noreply, state}
end
@impl true
def handle_continue(:ok, state) do
{:noreply, state}
end
@impl true
def handle_info(:consolidate, state) do
consolidate()
schedule_next_consolidation()
{:noreply, state}
end
@doc """
Fetches the info for a specific item from the `:ets` table.
"""
def fetch do
do_fetch(:ets.lookup(table_name(), cache_key()))
end
defp do_fetch([{_, result}]), do: result
defp do_fetch([]), do: 0
@doc """
Consolidates the info by populating the `:ets` table with the current database information.
"""
def consolidate do
counter = Chain.count_addresses()
insert_counter({cache_key(), counter})
end
@doc """
Returns a boolean that indicates whether consolidation is enabled
In order to choose whether or not to enable the scheduler and the initial
consolidation, change the following Explorer config:
`config :explorer, Explorer.Counters.AddressesCounter, enable_consolidation: true`
to:
`config :explorer, Explorer.Counters.AddressesCounter, enable_consolidation: false`
"""
def enable_consolidation?, do: @enable_consolidation
end

@ -27,6 +27,7 @@ defmodule Explorer.ChainTest do
alias Explorer.Chain.Supply.ProofOfAuthority
alias Explorer.Counters.AddressesWithBalanceCounter
alias Explorer.Counters.AddressesCounter
doctest Explorer.Chain
@ -50,6 +51,22 @@ defmodule Explorer.ChainTest do
end
end
describe "count_addresses_from_cache/0" do
test "returns the number of all addresses" do
insert(:address, fetched_coin_balance: 0)
insert(:address, fetched_coin_balance: 1)
insert(:address, fetched_coin_balance: 2)
start_supervised!(AddressesCounter)
AddressesCounter.consolidate()
addresses_with_balance = Chain.count_addresses_from_cache()
assert is_integer(addresses_with_balance)
assert addresses_with_balance == 3
end
end
describe "last_db_block_status/0" do
test "return no_blocks errors if db is empty" do
assert {:error, :no_blocks} = Chain.last_db_block_status()

@ -0,0 +1,16 @@
defmodule Explorer.Counters.AddressesCounterTest do
use Explorer.DataCase
alias Explorer.Counters.AddressesCounter
test "populates the cache with the number of all addresses" do
insert(:address, fetched_coin_balance: 0)
insert(:address, fetched_coin_balance: 1)
insert(:address, fetched_coin_balance: 2)
start_supervised!(AddressesCounter)
AddressesCounter.consolidate()
assert AddressesCounter.fetch() == 3
end
end
Loading…
Cancel
Save