Consolidate Token Holders count info

The number of token holders is now available in the token page again. It
is cached by the TokenHoldersCounter GenServer that periodically
updateds an ETS with all pairs of tokens and token holders.

apps/block_scout_web/lib/block_scout_web/controllers/tokens/holder_controller.ex

@@ -22,6 +22,7 @@ defmodule BlockScoutWeb.Tokens.HolderController do
         token: token,
         token_balances: token_balances_paginated,
         total_token_transfers: Chain.count_token_transfers_from_token_hash(address_hash),
+        total_token_holders: Chain.count_token_holders_from_token_hash(address_hash),
         next_page_params: next_page_params(next_page, token_balances_paginated, params)
       )
     else

apps/block_scout_web/lib/block_scout_web/controllers/tokens/inventory_controller.ex

@@ -23,6 +23,7 @@ defmodule BlockScoutWeb.Tokens.InventoryController do
         token: token,
         unique_tokens: unique_tokens_paginated,
         total_token_transfers: Chain.count_token_transfers_from_token_hash(address_hash),
+        total_token_holders: Chain.count_token_holders_from_token_hash(address_hash),
         next_page_params: unique_tokens_next_page(next_page, unique_tokens_paginated, params)
       )
     else

apps/block_scout_web/lib/block_scout_web/controllers/tokens/read_contract_controller.ex

@@ -10,7 +10,8 @@ defmodule BlockScoutWeb.Tokens.ReadContractController do
         conn,
         "index.html",
         token: token,
-        total_token_transfers: Chain.count_token_transfers_from_token_hash(address_hash)
+        total_token_transfers: Chain.count_token_transfers_from_token_hash(address_hash),
+        total_token_holders: Chain.count_token_holders_from_token_hash(address_hash)
       )
     else
       :error ->

apps/block_scout_web/lib/block_scout_web/controllers/tokens/transfer_controller.ex

@@ -17,6 +17,7 @@ defmodule BlockScoutWeb.Tokens.TransferController do
         transfers: token_transfers_paginated,
         token: token,
         total_token_transfers: Chain.count_token_transfers_from_token_hash(address_hash),
+        total_token_holders: Chain.count_token_holders_from_token_hash(address_hash),
         next_page_params: next_page_params(next_page, token_transfers_paginated, params)
       )
     else

apps/block_scout_web/lib/block_scout_web/templates/tokens/holder/index.html.eex

@@ -4,6 +4,7 @@
     "_details.html",
     token: @token,
     total_token_transfers: @total_token_transfers,
+    total_token_holders: @total_token_holders,
     conn: @conn
   ) %>
 

apps/block_scout_web/lib/block_scout_web/templates/tokens/inventory/index.html.eex

@@ -4,6 +4,7 @@
     "_details.html",
     token: @token,
     total_token_transfers: @total_token_transfers,
+    total_token_holders: @total_token_holders,
     conn: @conn
   ) %>
 

apps/block_scout_web/lib/block_scout_web/templates/tokens/overview/_details.html.eex

@@ -36,6 +36,7 @@
             </span>
             <div class="d-flex flex-row justify-content-start text-muted">
               <span class="mr-4"> <%= @token.type %> </span>
+              <span class="mr-4"><%= @total_token_holders %>  <%= gettext "addresses" %></span>
               <span class="mr-4"><%= @total_token_transfers %> <%= gettext "Transfers" %></span>
               <%= if decimals?(@token) do %>
                 <span class="mr-4"><%= @token.decimals %> <%= gettext "decimals" %></span>

apps/block_scout_web/lib/block_scout_web/templates/tokens/read_contract/index.html.eex

@@ -4,6 +4,7 @@
     "_details.html",
     token: @token,
     total_token_transfers: @total_token_transfers,
+    total_token_holders: @total_token_holders,
     conn: @conn
   ) %>
 

apps/block_scout_web/lib/block_scout_web/templates/tokens/token/show.html.eex

@@ -4,6 +4,7 @@
     "_details.html",
     token: @token,
     total_token_transfers: @total_token_transfers,
+    total_token_holders: @total_token_holders,
     conn: @conn
   ) %>
 

apps/block_scout_web/lib/block_scout_web/templates/tokens/transfer/index.html.eex

@@ -4,6 +4,7 @@
     "_details.html",
     token: @token,
     total_token_transfers: @total_token_transfers,
+    total_token_holders: @total_token_holders,
     conn: @conn
   ) %>
 

apps/block_scout_web/test/block_scout_web/controllers/tokens/read_contract_controller_test.exs

@@ -31,6 +31,7 @@ defmodule BlockScoutWeb.Tokens.ReadContractControllerTest do
       assert html_response(conn, 200)
       assert token.contract_address_hash == conn.assigns.token.contract_address_hash
       assert conn.assigns.total_token_transfers
+      assert conn.assigns.total_token_holders
     end
   end
 end

apps/explorer/config/config.exs

@@ -26,6 +26,8 @@ config :explorer, Explorer.Repo,
 
 config :explorer, Explorer.Counters.TokenTransferCounter, enabled: true
 
+config :explorer, Explorer.Counters.TokenHoldersCounter, enabled: true, enable_consolidation: true
+
 config :explorer,
   solc_bin_api_url: "https://solc-bin.ethereum.org"
 

apps/explorer/config/test.exs

@@ -17,6 +17,8 @@ config :explorer, Explorer.ExchangeRates, enabled: false
 
 config :explorer, Explorer.Market.History.Cataloger, enabled: false
 
+config :explorer, Explorer.Counters.TokenHoldersCounter, enabled: true, enable_consolidation: false
+
 config :logger, :explorer,
   level: :warn,
   path: Path.absname("logs/test/explorer.log")

apps/explorer/lib/explorer/application.ex

@@ -30,6 +30,7 @@ defmodule Explorer.Application do
     [
       configure(Explorer.ExchangeRates),
       configure(Explorer.Market.History.Cataloger),
+      configure(Explorer.Counters.TokenHoldersCounter),
       configure(Explorer.Counters.TokenTransferCounter),
       configure(Explorer.Counters.BlockValidationCounter)
     ]

apps/explorer/lib/explorer/chain.ex

@@ -37,7 +37,7 @@ defmodule Explorer.Chain do
 
   alias Explorer.Chain.Block.Reward
   alias Explorer.{PagingOptions, Repo}
-  alias Explorer.Counters.{TokenTransferCounter, BlockValidationCounter}
+  alias Explorer.Counters.{TokenHoldersCounter, TokenTransferCounter, BlockValidationCounter}
 
   @default_paging_options %PagingOptions{page_size: 50}
 
@@ -2038,14 +2038,9 @@ defmodule Explorer.Chain do
     |> Repo.all()
   end
 
-  # This function is deprecated.
-  #
-  # The code is being treated at https://github.com/poanetwork/blockscout/issues/880
   @spec count_token_holders_from_token_hash(Hash.Address.t()) :: non_neg_integer()
   def count_token_holders_from_token_hash(contract_address_hash) do
-    contract_address_hash
+    TokenHoldersCounter.fetch(contract_address_hash)
-    |> TokenBalance.token_holders_from_token_hash()
-    |> Repo.aggregate(:count, :address_hash)
   end
 
   @spec address_to_unique_tokens(Hash.Address.t(), [paging_options]) :: [TokenTransfer.t()]

apps/explorer/lib/explorer/chain/address/token_balance.ex

@@ -121,6 +121,29 @@ defmodule Explorer.Chain.Address.TokenBalance do
     )
   end
 
+  @doc """
+  Builds an `Ecto.Query` to group all tokens with their number of holders.
+  """
+  def tokens_grouped_by_number_of_holders do
+    query = unique_holders()
+
+    from(
+      tb in subquery(query),
+      where: tb.value > 0,
+      select: {tb.token_contract_address_hash, count(tb.address_hash)},
+      group_by: tb.token_contract_address_hash
+    )
+  end
+
+  defp unique_holders do
+    from(
+      tb in TokenBalance,
+      distinct: [:address_hash, :token_contract_address_hash],
+      where: tb.address_hash != ^@burn_address_hash,
+      order_by: [desc: :block_number]
+    )
+  end
+
   defp page_token_balances(query, %PagingOptions{key: nil}), do: query
 
   defp page_token_balances(query, %PagingOptions{key: {value, address_hash}}) do

apps/explorer/lib/explorer/counters/token_holders_counter.ex

@@ -0,0 +1,108 @@
+defmodule Explorer.Counters.TokenHoldersCounter do
+  use GenServer
+
+  @moduledoc """
+  Caches the number of token holders of a token.
+  """
+
+  alias Explorer.Chain.Address.TokenBalance
+  alias Explorer.Repo
+
+  @table :token_holders_counter
+
+  config = Application.get_env(:explorer, Explorer.Counters.TokenHoldersCounter)
+  @enable_consolidation Keyword.get(config, :enable_consolidation)
+
+  def table_name do
+    @table
+  end
+
+  @doc """
+  Starts a process to periodically update the counter of the token holders.
+  """
+  @spec start_link(term()) :: GenServer.on_start()
+  def start_link(_) do
+    GenServer.start_link(__MODULE__, :ok, name: __MODULE__)
+  end
+
+  ## Server
+  @impl true
+  def init(args) do
+    create_table()
+
+    if enable_consolidation?() do
+      Task.start_link(&consolidate/0)
+      schedule_next_consolidation()
+    end
+
+    {:ok, args}
+  end
+
+  def create_table do
+    opts = [
+      :set,
+      :named_table,
+      :public,
+      read_concurrency: true
+    ]
+
+    :ets.new(table_name(), opts)
+  end
+
+  @doc """
+  Consolidates the token holders info, by populating the `:ets` table with the current database information.
+  """
+  def consolidate do
+    TokenBalance.tokens_grouped_by_number_of_holders()
+    |> Repo.all()
+    |> Enum.map(fn {token, number_of_holders} ->
+      {token.bytes, number_of_holders}
+    end)
+    |> insert_counter()
+  end
+
+  defp schedule_next_consolidation do
+    if enable_consolidation?() do
+      # Schedule next consolidation to be run in 30 minutes
+      Process.send_after(self(), :consolidate, 30 * 60 * 1000)
+    end
+  end
+
+  @doc """
+  Fetches the token holders info for a specific token from the `:ets` table.
+  """
+  def fetch(token_hash) do
+    do_fetch(:ets.lookup(table_name(), token_hash.bytes))
+  end
+
+  defp do_fetch([{_, result}]), do: result
+  defp do_fetch([]), do: 0
+
+  @doc """
+  Inserts new items into the `:ets` table.
+  """
+  def insert_counter(token_holders) do
+    :ets.insert(table_name(), token_holders)
+  end
+
+  @impl true
+  def handle_info(:consolidate, state) do
+    consolidate()
+
+    schedule_next_consolidation()
+
+    {:noreply, state}
+  end
+
+  # We don't want to automatically start the consolidation in all environments.
+  # Consider the test environment: if the consolidation initiates but does not
+  # finishes before a test ends, that test will fail. This way, hundreds o
+  # tests were failing before disabling the consolidation and the scheduler in
+  # the test env.
+  #
+  # In order to choose whether or not to enable the scheduler and the initial
+  # consolidation, change the following Explorer config:
+  #
+  # config :explorer, Explorer.Counters.TokenHoldersCounter, enable_consolidation: false
+  defp enable_consolidation?, do: @enable_consolidation
+end

apps/explorer/test/explorer/chain/address/token_balance_test.exs

@@ -2,6 +2,7 @@ defmodule Explorer.Chain.Address.TokenBalanceTest do
   use Explorer.DataCase
 
   alias Explorer.Repo
+  alias Explorer.Chain.Token
   alias Explorer.Chain.Address.TokenBalance
 
   describe "unfetched_token_balances/0" do
@@ -76,4 +77,148 @@ defmodule Explorer.Chain.Address.TokenBalanceTest do
       assert result.block_number == token_balance.block_number
     end
   end
+
+  describe "tokens_grouped_by_number_of_holders/0" do
+    test "groups all tokens with their number of holders" do
+      token_a = insert(:token)
+      address_a = insert(:address, hash: "0xc45e4830dff873cf8b70de2b194d0ddd06ef651d")
+
+      insert(:token_balance, address: address_a, value: 10, token_contract_address_hash: token_a.contract_address_hash)
+
+      token_b = insert(:token)
+      address_b = insert(:address, hash: "0xc45e4830dff873cf8b70de2b194d0ddd06ef651e")
+      address_c = insert(:address, hash: "0xc45e4830dff873cf8b70de2b194d0ddd06ef651f")
+
+      insert(
+        :token_balance,
+        address: address_b,
+        value: 10,
+        token_contract_address_hash: token_b.contract_address_hash
+      )
+
+      insert(
+        :token_balance,
+        address: address_c,
+        value: 10,
+        token_contract_address_hash: token_b.contract_address_hash
+      )
+
+      result =
+        TokenBalance.tokens_grouped_by_number_of_holders()
+        |> Repo.all()
+        |> Enum.sort(fn {_token_hash_a, holders_a}, {_token_hash_b, holders_b} ->
+          holders_a < holders_b
+        end)
+
+      assert [{token_a_result, 1}, {token_b_result, 2}] = result
+      assert token_a_result == token_a.contract_address_hash
+      assert token_b_result == token_b.contract_address_hash
+    end
+
+    test "considers only the last block" do
+      address = insert(:address, hash: "0xe49fedd93960a0267b3c3b2c1e2d66028e013fee")
+
+      %Token{contract_address_hash: contract_address_hash} = insert(:token)
+
+      insert(
+        :token_balance,
+        address: address,
+        block_number: 1000,
+        token_contract_address_hash: contract_address_hash,
+        value: 5000
+      )
+
+      insert(
+        :token_balance,
+        address: address,
+        block_number: 1002,
+        token_contract_address_hash: contract_address_hash,
+        value: 1000
+      )
+
+      [{_, result}] = Repo.all(TokenBalance.tokens_grouped_by_number_of_holders())
+
+      assert result == 1
+    end
+
+    test "counts only the last block that has value greater than 0" do
+      address = insert(:address, hash: "0xe49fedd93960a0267b3c3b2c1e2d66028e013fee")
+
+      %Token{contract_address_hash: contract_address_hash} = insert(:token)
+
+      insert(
+        :token_balance,
+        address: address,
+        block_number: 1000,
+        token_contract_address_hash: contract_address_hash,
+        value: 5000
+      )
+
+      insert(
+        :token_balance,
+        address: address,
+        block_number: 1002,
+        token_contract_address_hash: contract_address_hash,
+        value: 0
+      )
+
+      result =
+        TokenBalance.tokens_grouped_by_number_of_holders()
+        |> Repo.all()
+        |> Enum.count()
+
+      assert result == 0
+    end
+
+    test "does not consider the burn address" do
+      burn_address = insert(:address, hash: "0x0000000000000000000000000000000000000000")
+
+      %Token{contract_address_hash: contract_address_hash} = insert(:token)
+
+      insert(
+        :token_balance,
+        address: burn_address,
+        block_number: 1000,
+        token_contract_address_hash: contract_address_hash,
+        value: 5000
+      )
+
+      result =
+        TokenBalance.tokens_grouped_by_number_of_holders()
+        |> Repo.all()
+        |> Enum.count()
+
+      assert result == 0
+    end
+
+    test "considers the same address for different tokens" do
+      address = insert(:address, hash: "0xe49fedd93960a0267b3c3b2c1e2d66028e013fee")
+
+      %Token{contract_address_hash: contract_address_hash_1} = insert(:token)
+      %Token{contract_address_hash: contract_address_hash_2} = insert(:token)
+
+      insert(
+        :token_balance,
+        address: address,
+        block_number: 1000,
+        token_contract_address_hash: contract_address_hash_1,
+        value: 5000
+      )
+
+      insert(
+        :token_balance,
+        address: address,
+        block_number: 1002,
+        token_contract_address_hash: contract_address_hash_2,
+        value: 5000
+      )
+
+      result =
+        TokenBalance.tokens_grouped_by_number_of_holders()
+        |> Repo.all()
+        |> Enum.count()
+
+      assert result == 2
+    end
+  end
 end

apps/explorer/test/explorer/chain_test.exs

@@ -24,6 +24,8 @@ defmodule Explorer.ChainTest do
 
   alias Explorer.Chain.Supply.ProofOfAuthority
 
+  alias Explorer.Counters.TokenHoldersCounter
+
   doctest Explorer.Chain
 
   describe "address_estimated_count/1" do
@@ -2860,7 +2862,7 @@ defmodule Explorer.ChainTest do
   end
 
   describe "count_token_holders_from_token_hash" do
-    test "counts different addresses that have the token" do
+    test "returns the most current count about token holders" do
       address_a = insert(:address, hash: "0xe49fedd93960a0267b3c3b2c1e2d66028e013fee")
       address_b = insert(:address, hash: "0x5f26097334b6a32b7951df61fd0c5803ec5d8354")
 
@@ -2882,55 +2884,9 @@ defmodule Explorer.ChainTest do
         value: 1000
       )
 
-      assert Chain.count_token_holders_from_token_hash(contract_address_hash) == 2
+      TokenHoldersCounter.consolidate()
-    end
-
-    test "counts only the last block" do
-      address = insert(:address, hash: "0xe49fedd93960a0267b3c3b2c1e2d66028e013fee")
-
-      %Token{contract_address_hash: contract_address_hash} = insert(:token)
-
-      insert(
-        :token_balance,
-        address: address,
-        block_number: 1000,
-        token_contract_address_hash: contract_address_hash,
-        value: 5000
-      )
-
-      insert(
-        :token_balance,
-        address: address,
-        block_number: 1002,
-        token_contract_address_hash: contract_address_hash,
-        value: 1000
-      )
-
-      assert Chain.count_token_holders_from_token_hash(contract_address_hash) == 1
-    end
-
-    test "counts only the last block that has value greater than 0" do
-      address = insert(:address, hash: "0xe49fedd93960a0267b3c3b2c1e2d66028e013fee")
-
-      %Token{contract_address_hash: contract_address_hash} = insert(:token)
-
-      insert(
-        :token_balance,
-        address: address,
-        block_number: 1000,
-        token_contract_address_hash: contract_address_hash,
-        value: 5000
-      )
-
-      insert(
-        :token_balance,
-        address: address,
-        block_number: 1002,
-        token_contract_address_hash: contract_address_hash,
-        value: 0
-      )
 
-      assert Chain.count_token_holders_from_token_hash(contract_address_hash) == 0
+      assert Chain.count_token_holders_from_token_hash(contract_address_hash) == 2
     end
   end
 

apps/explorer/test/explorer/counters/token_holders_counter_test.exs

@@ -0,0 +1,51 @@
+defmodule Explorer.Counters.TokenHoldersCounterTest do
+  use Explorer.DataCase
+
+  alias Explorer.Chain.Token
+  alias Explorer.Counters.TokenHoldersCounter
+
+  describe "consolidate/0" do
+    test "consolidates the token holders info with the most current database info" do
+      address_a = insert(:address, hash: "0xe49fedd93960a0267b3c3b2c1e2d66028e013fee")
+      address_b = insert(:address, hash: "0x5f26097334b6a32b7951df61fd0c5803ec5d8354")
+
+      %Token{contract_address_hash: contract_address_hash} = insert(:token)
+
+      insert(
+        :token_balance,
+        address: address_a,
+        block_number: 1000,
+        token_contract_address_hash: contract_address_hash,
+        value: 5000
+      )
+
+      TokenHoldersCounter.consolidate()
+
+      assert TokenHoldersCounter.fetch(contract_address_hash) == 1
+
+      insert(
+        :token_balance,
+        address: address_b,
+        block_number: 1002,
+        token_contract_address_hash: contract_address_hash,
+        value: 1000
+      )
+
+      TokenHoldersCounter.consolidate()
+
+      assert TokenHoldersCounter.fetch(contract_address_hash) == 2
+    end
+  end
+
+  describe "fetch/1" do
+    test "fetchs the total token holders by token contract address hash" do
+      token = insert(:token)
+
+      assert TokenHoldersCounter.fetch(token.contract_address_hash) == 0
+
+      TokenHoldersCounter.insert_counter({token.contract_address_hash.bytes, 15})
+
+      assert TokenHoldersCounter.fetch(token.contract_address_hash) == 15
+    end
+  end
+end