Consolidate Token Holders count info

The number of token holders is now available in the token page again. It
is cached by the TokenHoldersCounter GenServer that periodically
updateds an ETS with all pairs of tokens and token holders.
pull/904/head
Lucas Narciso 6 years ago
parent 234128a923
commit ea3da7fd07
No known key found for this signature in database
GPG Key ID: 9E89F4CF3FBAB001
  1. 1
      apps/block_scout_web/lib/block_scout_web/controllers/tokens/holder_controller.ex
  2. 1
      apps/block_scout_web/lib/block_scout_web/controllers/tokens/inventory_controller.ex
  3. 3
      apps/block_scout_web/lib/block_scout_web/controllers/tokens/read_contract_controller.ex
  4. 1
      apps/block_scout_web/lib/block_scout_web/controllers/tokens/transfer_controller.ex
  5. 1
      apps/block_scout_web/lib/block_scout_web/templates/tokens/holder/index.html.eex
  6. 1
      apps/block_scout_web/lib/block_scout_web/templates/tokens/inventory/index.html.eex
  7. 1
      apps/block_scout_web/lib/block_scout_web/templates/tokens/overview/_details.html.eex
  8. 1
      apps/block_scout_web/lib/block_scout_web/templates/tokens/read_contract/index.html.eex
  9. 1
      apps/block_scout_web/lib/block_scout_web/templates/tokens/token/show.html.eex
  10. 1
      apps/block_scout_web/lib/block_scout_web/templates/tokens/transfer/index.html.eex
  11. 1
      apps/block_scout_web/test/block_scout_web/controllers/tokens/read_contract_controller_test.exs
  12. 2
      apps/explorer/config/config.exs
  13. 2
      apps/explorer/config/test.exs
  14. 1
      apps/explorer/lib/explorer/application.ex
  15. 9
      apps/explorer/lib/explorer/chain.ex
  16. 23
      apps/explorer/lib/explorer/chain/address/token_balance.ex
  17. 108
      apps/explorer/lib/explorer/counters/token_holders_counter.ex
  18. 145
      apps/explorer/test/explorer/chain/address/token_balance_test.exs
  19. 54
      apps/explorer/test/explorer/chain_test.exs
  20. 51
      apps/explorer/test/explorer/counters/token_holders_counter_test.exs

@ -22,6 +22,7 @@ defmodule BlockScoutWeb.Tokens.HolderController do
token: token,
token_balances: token_balances_paginated,
total_token_transfers: Chain.count_token_transfers_from_token_hash(address_hash),
total_token_holders: Chain.count_token_holders_from_token_hash(address_hash),
next_page_params: next_page_params(next_page, token_balances_paginated, params)
)
else

@ -23,6 +23,7 @@ defmodule BlockScoutWeb.Tokens.InventoryController do
token: token,
unique_tokens: unique_tokens_paginated,
total_token_transfers: Chain.count_token_transfers_from_token_hash(address_hash),
total_token_holders: Chain.count_token_holders_from_token_hash(address_hash),
next_page_params: unique_tokens_next_page(next_page, unique_tokens_paginated, params)
)
else

@ -10,7 +10,8 @@ defmodule BlockScoutWeb.Tokens.ReadContractController do
conn,
"index.html",
token: token,
total_token_transfers: Chain.count_token_transfers_from_token_hash(address_hash)
total_token_transfers: Chain.count_token_transfers_from_token_hash(address_hash),
total_token_holders: Chain.count_token_holders_from_token_hash(address_hash)
)
else
:error ->

@ -17,6 +17,7 @@ defmodule BlockScoutWeb.Tokens.TransferController do
transfers: token_transfers_paginated,
token: token,
total_token_transfers: Chain.count_token_transfers_from_token_hash(address_hash),
total_token_holders: Chain.count_token_holders_from_token_hash(address_hash),
next_page_params: next_page_params(next_page, token_transfers_paginated, params)
)
else

@ -4,6 +4,7 @@
"_details.html",
token: @token,
total_token_transfers: @total_token_transfers,
total_token_holders: @total_token_holders,
conn: @conn
) %>

@ -4,6 +4,7 @@
"_details.html",
token: @token,
total_token_transfers: @total_token_transfers,
total_token_holders: @total_token_holders,
conn: @conn
) %>

@ -36,6 +36,7 @@
</span>
<div class="d-flex flex-row justify-content-start text-muted">
<span class="mr-4"> <%= @token.type %> </span>
<span class="mr-4"><%= @total_token_holders %> <%= gettext "addresses" %></span>
<span class="mr-4"><%= @total_token_transfers %> <%= gettext "Transfers" %></span>
<%= if decimals?(@token) do %>
<span class="mr-4"><%= @token.decimals %> <%= gettext "decimals" %></span>

@ -4,6 +4,7 @@
"_details.html",
token: @token,
total_token_transfers: @total_token_transfers,
total_token_holders: @total_token_holders,
conn: @conn
) %>

@ -4,6 +4,7 @@
"_details.html",
token: @token,
total_token_transfers: @total_token_transfers,
total_token_holders: @total_token_holders,
conn: @conn
) %>

@ -4,6 +4,7 @@
"_details.html",
token: @token,
total_token_transfers: @total_token_transfers,
total_token_holders: @total_token_holders,
conn: @conn
) %>

@ -31,6 +31,7 @@ defmodule BlockScoutWeb.Tokens.ReadContractControllerTest do
assert html_response(conn, 200)
assert token.contract_address_hash == conn.assigns.token.contract_address_hash
assert conn.assigns.total_token_transfers
assert conn.assigns.total_token_holders
end
end
end

@ -26,6 +26,8 @@ config :explorer, Explorer.Repo,
config :explorer, Explorer.Counters.TokenTransferCounter, enabled: true
config :explorer, Explorer.Counters.TokenHoldersCounter, enabled: true, enable_consolidation: true
config :explorer,
solc_bin_api_url: "https://solc-bin.ethereum.org"

@ -17,6 +17,8 @@ config :explorer, Explorer.ExchangeRates, enabled: false
config :explorer, Explorer.Market.History.Cataloger, enabled: false
config :explorer, Explorer.Counters.TokenHoldersCounter, enabled: true, enable_consolidation: false
config :logger, :explorer,
level: :warn,
path: Path.absname("logs/test/explorer.log")

@ -30,6 +30,7 @@ defmodule Explorer.Application do
[
configure(Explorer.ExchangeRates),
configure(Explorer.Market.History.Cataloger),
configure(Explorer.Counters.TokenHoldersCounter),
configure(Explorer.Counters.TokenTransferCounter),
configure(Explorer.Counters.BlockValidationCounter)
]

@ -37,7 +37,7 @@ defmodule Explorer.Chain do
alias Explorer.Chain.Block.Reward
alias Explorer.{PagingOptions, Repo}
alias Explorer.Counters.{TokenTransferCounter, BlockValidationCounter}
alias Explorer.Counters.{TokenHoldersCounter, TokenTransferCounter, BlockValidationCounter}
@default_paging_options %PagingOptions{page_size: 50}
@ -2038,14 +2038,9 @@ defmodule Explorer.Chain do
|> Repo.all()
end
# This function is deprecated.
#
# The code is being treated at https://github.com/poanetwork/blockscout/issues/880
@spec count_token_holders_from_token_hash(Hash.Address.t()) :: non_neg_integer()
def count_token_holders_from_token_hash(contract_address_hash) do
contract_address_hash
|> TokenBalance.token_holders_from_token_hash()
|> Repo.aggregate(:count, :address_hash)
TokenHoldersCounter.fetch(contract_address_hash)
end
@spec address_to_unique_tokens(Hash.Address.t(), [paging_options]) :: [TokenTransfer.t()]

@ -121,6 +121,29 @@ defmodule Explorer.Chain.Address.TokenBalance do
)
end
@doc """
Builds an `Ecto.Query` to group all tokens with their number of holders.
"""
def tokens_grouped_by_number_of_holders do
query = unique_holders()
from(
tb in subquery(query),
where: tb.value > 0,
select: {tb.token_contract_address_hash, count(tb.address_hash)},
group_by: tb.token_contract_address_hash
)
end
defp unique_holders do
from(
tb in TokenBalance,
distinct: [:address_hash, :token_contract_address_hash],
where: tb.address_hash != ^@burn_address_hash,
order_by: [desc: :block_number]
)
end
defp page_token_balances(query, %PagingOptions{key: nil}), do: query
defp page_token_balances(query, %PagingOptions{key: {value, address_hash}}) do

@ -0,0 +1,108 @@
defmodule Explorer.Counters.TokenHoldersCounter do
use GenServer
@moduledoc """
Caches the number of token holders of a token.
"""
alias Explorer.Chain.Address.TokenBalance
alias Explorer.Repo
@table :token_holders_counter
config = Application.get_env(:explorer, Explorer.Counters.TokenHoldersCounter)
@enable_consolidation Keyword.get(config, :enable_consolidation)
def table_name do
@table
end
@doc """
Starts a process to periodically update the counter of the token holders.
"""
@spec start_link(term()) :: GenServer.on_start()
def start_link(_) do
GenServer.start_link(__MODULE__, :ok, name: __MODULE__)
end
## Server
@impl true
def init(args) do
create_table()
if enable_consolidation?() do
Task.start_link(&consolidate/0)
schedule_next_consolidation()
end
{:ok, args}
end
def create_table do
opts = [
:set,
:named_table,
:public,
read_concurrency: true
]
:ets.new(table_name(), opts)
end
@doc """
Consolidates the token holders info, by populating the `:ets` table with the current database information.
"""
def consolidate do
TokenBalance.tokens_grouped_by_number_of_holders()
|> Repo.all()
|> Enum.map(fn {token, number_of_holders} ->
{token.bytes, number_of_holders}
end)
|> insert_counter()
end
defp schedule_next_consolidation do
if enable_consolidation?() do
# Schedule next consolidation to be run in 30 minutes
Process.send_after(self(), :consolidate, 30 * 60 * 1000)
end
end
@doc """
Fetches the token holders info for a specific token from the `:ets` table.
"""
def fetch(token_hash) do
do_fetch(:ets.lookup(table_name(), token_hash.bytes))
end
defp do_fetch([{_, result}]), do: result
defp do_fetch([]), do: 0
@doc """
Inserts new items into the `:ets` table.
"""
def insert_counter(token_holders) do
:ets.insert(table_name(), token_holders)
end
@impl true
def handle_info(:consolidate, state) do
consolidate()
schedule_next_consolidation()
{:noreply, state}
end
# We don't want to automatically start the consolidation in all environments.
# Consider the test environment: if the consolidation initiates but does not
# finishes before a test ends, that test will fail. This way, hundreds o
# tests were failing before disabling the consolidation and the scheduler in
# the test env.
#
# In order to choose whether or not to enable the scheduler and the initial
# consolidation, change the following Explorer config:
#
# config :explorer, Explorer.Counters.TokenHoldersCounter, enable_consolidation: false
defp enable_consolidation?, do: @enable_consolidation
end

@ -2,6 +2,7 @@ defmodule Explorer.Chain.Address.TokenBalanceTest do
use Explorer.DataCase
alias Explorer.Repo
alias Explorer.Chain.Token
alias Explorer.Chain.Address.TokenBalance
describe "unfetched_token_balances/0" do
@ -76,4 +77,148 @@ defmodule Explorer.Chain.Address.TokenBalanceTest do
assert result.block_number == token_balance.block_number
end
end
describe "tokens_grouped_by_number_of_holders/0" do
test "groups all tokens with their number of holders" do
token_a = insert(:token)
address_a = insert(:address, hash: "0xc45e4830dff873cf8b70de2b194d0ddd06ef651d")
insert(:token_balance, address: address_a, value: 10, token_contract_address_hash: token_a.contract_address_hash)
token_b = insert(:token)
address_b = insert(:address, hash: "0xc45e4830dff873cf8b70de2b194d0ddd06ef651e")
address_c = insert(:address, hash: "0xc45e4830dff873cf8b70de2b194d0ddd06ef651f")
insert(
:token_balance,
address: address_b,
value: 10,
token_contract_address_hash: token_b.contract_address_hash
)
insert(
:token_balance,
address: address_c,
value: 10,
token_contract_address_hash: token_b.contract_address_hash
)
result =
TokenBalance.tokens_grouped_by_number_of_holders()
|> Repo.all()
|> Enum.sort(fn {_token_hash_a, holders_a}, {_token_hash_b, holders_b} ->
holders_a < holders_b
end)
assert [{token_a_result, 1}, {token_b_result, 2}] = result
assert token_a_result == token_a.contract_address_hash
assert token_b_result == token_b.contract_address_hash
end
test "considers only the last block" do
address = insert(:address, hash: "0xe49fedd93960a0267b3c3b2c1e2d66028e013fee")
%Token{contract_address_hash: contract_address_hash} = insert(:token)
insert(
:token_balance,
address: address,
block_number: 1000,
token_contract_address_hash: contract_address_hash,
value: 5000
)
insert(
:token_balance,
address: address,
block_number: 1002,
token_contract_address_hash: contract_address_hash,
value: 1000
)
[{_, result}] = Repo.all(TokenBalance.tokens_grouped_by_number_of_holders())
assert result == 1
end
test "counts only the last block that has value greater than 0" do
address = insert(:address, hash: "0xe49fedd93960a0267b3c3b2c1e2d66028e013fee")
%Token{contract_address_hash: contract_address_hash} = insert(:token)
insert(
:token_balance,
address: address,
block_number: 1000,
token_contract_address_hash: contract_address_hash,
value: 5000
)
insert(
:token_balance,
address: address,
block_number: 1002,
token_contract_address_hash: contract_address_hash,
value: 0
)
result =
TokenBalance.tokens_grouped_by_number_of_holders()
|> Repo.all()
|> Enum.count()
assert result == 0
end
test "does not consider the burn address" do
burn_address = insert(:address, hash: "0x0000000000000000000000000000000000000000")
%Token{contract_address_hash: contract_address_hash} = insert(:token)
insert(
:token_balance,
address: burn_address,
block_number: 1000,
token_contract_address_hash: contract_address_hash,
value: 5000
)
result =
TokenBalance.tokens_grouped_by_number_of_holders()
|> Repo.all()
|> Enum.count()
assert result == 0
end
test "considers the same address for different tokens" do
address = insert(:address, hash: "0xe49fedd93960a0267b3c3b2c1e2d66028e013fee")
%Token{contract_address_hash: contract_address_hash_1} = insert(:token)
%Token{contract_address_hash: contract_address_hash_2} = insert(:token)
insert(
:token_balance,
address: address,
block_number: 1000,
token_contract_address_hash: contract_address_hash_1,
value: 5000
)
insert(
:token_balance,
address: address,
block_number: 1002,
token_contract_address_hash: contract_address_hash_2,
value: 5000
)
result =
TokenBalance.tokens_grouped_by_number_of_holders()
|> Repo.all()
|> Enum.count()
assert result == 2
end
end
end

@ -24,6 +24,8 @@ defmodule Explorer.ChainTest do
alias Explorer.Chain.Supply.ProofOfAuthority
alias Explorer.Counters.TokenHoldersCounter
doctest Explorer.Chain
describe "address_estimated_count/1" do
@ -2860,7 +2862,7 @@ defmodule Explorer.ChainTest do
end
describe "count_token_holders_from_token_hash" do
test "counts different addresses that have the token" do
test "returns the most current count about token holders" do
address_a = insert(:address, hash: "0xe49fedd93960a0267b3c3b2c1e2d66028e013fee")
address_b = insert(:address, hash: "0x5f26097334b6a32b7951df61fd0c5803ec5d8354")
@ -2882,55 +2884,9 @@ defmodule Explorer.ChainTest do
value: 1000
)
assert Chain.count_token_holders_from_token_hash(contract_address_hash) == 2
end
test "counts only the last block" do
address = insert(:address, hash: "0xe49fedd93960a0267b3c3b2c1e2d66028e013fee")
%Token{contract_address_hash: contract_address_hash} = insert(:token)
insert(
:token_balance,
address: address,
block_number: 1000,
token_contract_address_hash: contract_address_hash,
value: 5000
)
insert(
:token_balance,
address: address,
block_number: 1002,
token_contract_address_hash: contract_address_hash,
value: 1000
)
assert Chain.count_token_holders_from_token_hash(contract_address_hash) == 1
end
test "counts only the last block that has value greater than 0" do
address = insert(:address, hash: "0xe49fedd93960a0267b3c3b2c1e2d66028e013fee")
%Token{contract_address_hash: contract_address_hash} = insert(:token)
insert(
:token_balance,
address: address,
block_number: 1000,
token_contract_address_hash: contract_address_hash,
value: 5000
)
insert(
:token_balance,
address: address,
block_number: 1002,
token_contract_address_hash: contract_address_hash,
value: 0
)
TokenHoldersCounter.consolidate()
assert Chain.count_token_holders_from_token_hash(contract_address_hash) == 0
assert Chain.count_token_holders_from_token_hash(contract_address_hash) == 2
end
end

@ -0,0 +1,51 @@
defmodule Explorer.Counters.TokenHoldersCounterTest do
use Explorer.DataCase
alias Explorer.Chain.Token
alias Explorer.Counters.TokenHoldersCounter
describe "consolidate/0" do
test "consolidates the token holders info with the most current database info" do
address_a = insert(:address, hash: "0xe49fedd93960a0267b3c3b2c1e2d66028e013fee")
address_b = insert(:address, hash: "0x5f26097334b6a32b7951df61fd0c5803ec5d8354")
%Token{contract_address_hash: contract_address_hash} = insert(:token)
insert(
:token_balance,
address: address_a,
block_number: 1000,
token_contract_address_hash: contract_address_hash,
value: 5000
)
TokenHoldersCounter.consolidate()
assert TokenHoldersCounter.fetch(contract_address_hash) == 1
insert(
:token_balance,
address: address_b,
block_number: 1002,
token_contract_address_hash: contract_address_hash,
value: 1000
)
TokenHoldersCounter.consolidate()
assert TokenHoldersCounter.fetch(contract_address_hash) == 2
end
end
describe "fetch/1" do
test "fetchs the total token holders by token contract address hash" do
token = insert(:token)
assert TokenHoldersCounter.fetch(token.contract_address_hash) == 0
TokenHoldersCounter.insert_counter({token.contract_address_hash.bytes, 15})
assert TokenHoldersCounter.fetch(token.contract_address_hash) == 15
end
end
end
Loading…
Cancel
Save