Merge pull request #1170 from poanetwork/ams-address-count

Add a cache to count the addresses with balance > 0.

apps/block_scout_web/lib/block_scout_web/controllers/address_controller.ex

@@ -8,7 +8,7 @@ defmodule BlockScoutWeb.AddressController do
   def index(conn, _params) do
     render(conn, "index.html",
       address_tx_count_pairs: Chain.list_top_addresses(),
-      address_estimated_count: Chain.address_estimated_count(),
+      address_count: Chain.count_addresses_with_balance_from_cache(),
       exchange_rate: Market.get_exchange_rate(Explorer.coin()) || Token.null(),
       total_supply: Chain.total_supply()
     )

apps/block_scout_web/lib/block_scout_web/controllers/chain_controller.ex

@@ -36,7 +36,7 @@ defmodule BlockScoutWeb.ChainController do
     render(
       conn,
       "show.html",
-      address_estimated_count: Chain.address_estimated_count(),
+      address_count: Chain.count_addresses_with_balance_from_cache(),
       average_block_time: Chain.average_block_time(),
       blocks: blocks,
       exchange_rate: exchange_rate,

apps/block_scout_web/lib/block_scout_web/notifier.ex

@@ -10,7 +10,7 @@ defmodule BlockScoutWeb.Notifier do
   alias Explorer.ExchangeRates.Token
 
   def handle_event({:chain_event, :addresses, :realtime, addresses}) do
-    Endpoint.broadcast("addresses:new_address", "count", %{count: Chain.address_estimated_count()})
+    Endpoint.broadcast("addresses:new_address", "count", %{count: Chain.count_addresses_with_balance_from_cache()})
 
     addresses
     |> Stream.reject(fn %Address{fetched_coin_balance: fetched_coin_balance} -> is_nil(fetched_coin_balance) end)

apps/block_scout_web/lib/block_scout_web/templates/address/index.html.eex

@@ -4,7 +4,7 @@
       <h1><%= gettext "Addresses" %></h1>
       <p>
         <%= gettext "Showing 250 addresses of" %>
-        <%= Cldr.Number.to_string!(@address_estimated_count, format: "#,###") %>
+        <%= Cldr.Number.to_string!(@address_count, format: "#,###") %>
         <%= gettext "total addresses with a balance" %>
       </p>
 

apps/block_scout_web/lib/block_scout_web/templates/chain/show.html.eex

@@ -43,7 +43,7 @@
             <%= gettext "Wallet addresses" %>
           </span>
           <span class="dashboard-banner-network-stats-value" data-selector="address-count">
-            <%= Cldr.Number.to_string!(@address_estimated_count, format: "#,###") %>
+            <%= Cldr.Number.to_string!(@address_count, format: "#,###") %>
           </span>
         </div>
       </div>

apps/explorer/config/config.exs

@@ -17,8 +17,6 @@ config :explorer, Explorer.Integrations.EctoLogger, query_time_ms_threshold: 2_0
 
 config :explorer, Explorer.ExchangeRates, enabled: true
 
-config :explorer, Explorer.Counters.BlockValidationCounter, enabled: true
-
 config :explorer, Explorer.Market.History.Cataloger, enabled: true
 
 config :explorer, Explorer.Repo,
@@ -32,8 +30,12 @@ config :explorer, Explorer.Tracer,
 
 config :explorer, Explorer.Counters.TokenTransferCounter, enabled: true
 
+config :explorer, Explorer.Counters.BlockValidationCounter, enabled: true, enable_consolidation: true
+
 config :explorer, Explorer.Counters.TokenHoldersCounter, enabled: true, enable_consolidation: true
 
+config :explorer, Explorer.Counters.AddessesWithBalanceCounter, enabled: true, enable_consolidation: true
+
 if System.get_env("SUPPLY_MODULE") == "TransactionAndLog" do
   config :explorer, supply: Explorer.Chain.Supply.TransactionAndLog
 end

apps/explorer/config/test.exs

@@ -3,8 +3,6 @@ use Mix.Config
 # Lower hashing rounds for faster tests
 config :bcrypt_elixir, log_rounds: 4
 
-config :explorer, Explorer.Counters.TokenHoldersCounter, enabled: true, enable_consolidation: false
-
 # Configure your database
 config :explorer, Explorer.Repo,
   adapter: Ecto.Adapters.Postgres,
@@ -21,6 +19,12 @@ config :explorer, Explorer.Market.History.Cataloger, enabled: false
 
 config :explorer, Explorer.Tracer, disabled?: false
 
+config :explorer, Explorer.Counters.BlockValidationCounter, enabled: true, enable_consolidation: false
+
+config :explorer, Explorer.Counters.TokenHoldersCounter, enabled: true, enable_consolidation: false
+
+config :explorer, Explorer.Counters.AddessesWithBalanceCounter, enabled: true, enable_consolidation: false
+
 config :logger, :explorer,
   level: :warn,
   path: Path.absname("logs/test/explorer.log")

apps/explorer/lib/explorer/application.ex

@@ -35,7 +35,8 @@ defmodule Explorer.Application do
       configure(Explorer.Market.History.Cataloger),
       configure(Explorer.Counters.TokenHoldersCounter),
       configure(Explorer.Counters.TokenTransferCounter),
-      configure(Explorer.Counters.BlockValidationCounter)
+      configure(Explorer.Counters.BlockValidationCounter),
+      configure(Explorer.Counters.AddessesWithBalanceCounter)
     ]
     |> List.flatten()
   end

apps/explorer/lib/explorer/chain.ex

@@ -40,7 +40,13 @@ defmodule Explorer.Chain do
 
   alias Explorer.Chain.Block.Reward
   alias Explorer.{PagingOptions, Repo}
-  alias Explorer.Counters.{BlockValidationCounter, TokenHoldersCounter, TokenTransferCounter}
+
+  alias Explorer.Counters.{
+    AddessesWithBalanceCounter,
+    BlockValidationCounter,
+    TokenHoldersCounter,
+    TokenTransferCounter
+  }
 
   alias Dataloader.Ecto, as: DataloaderEcto
 
@@ -82,18 +88,24 @@ defmodule Explorer.Chain do
   @typep paging_options :: {:paging_options, PagingOptions.t()}
 
   @doc """
-  Gets an estimated count of `t:Explorer.Chain.Address.t/0`'s where the `fetched_coin_balance` is > 0
+  Gets from the cache the count of `t:Explorer.Chain.Address.t/0`'s where the `fetched_coin_balance` is > 0
   """
-  @spec address_estimated_count :: non_neg_integer()
+  @spec count_addresses_with_balance_from_cache :: non_neg_integer()
-  def address_estimated_count do
+  def count_addresses_with_balance_from_cache do
-    {:ok, %Postgrex.Result{rows: result}} =
+    AddessesWithBalanceCounter.fetch()
-      Repo.query("""
+  end
-      EXPLAIN SELECT COUNT(a0.hash) FROM addresses AS a0 WHERE (a0.fetched_coin_balance > 0)
+
-      """)
+  @doc """
+  Counts the number of addresses with fetched coin balance > 0.
 
-    {[explain], _} = List.pop_at(result, 1)
+  This function should be used with caution. In larger databases, it may take a
-    [[_ | [rows]]] = Regex.scan(~r/rows=(\d+)/, explain)
+  while to have the return back.
-    String.to_integer(rows)
+  """
+  def count_addresses_with_balance do
+    Repo.one(
+      Address.count_with_fetched_coin_balance(),
+      timeout: :infinity
+    )
   end
 
   @doc """

apps/explorer/lib/explorer/chain/address.ex

@@ -104,4 +104,15 @@ defmodule Explorer.Chain.Address do
       @protocol.to_string(hash)
     end
   end
+
+  @doc """
+  Counts all the addresses where the `fetched_coin_balance` is > 0.
+  """
+  def count_with_fetched_coin_balance do
+    from(
+      a in Address,
+      select: fragment("COUNT(*)"),
+      where: a.fetched_coin_balance > ^0
+    )
+  end
 end

apps/explorer/lib/explorer/counters/addresses_with_balance_counter.ex

@@ -0,0 +1,117 @@
+defmodule Explorer.Counters.AddessesWithBalanceCounter do
+  @moduledoc """
+  Caches the number of addresses with fetched coin balance > 0.
+
+  It loads the count asynchronously and in a time interval of 30 minutes.
+  """
+
+  use GenServer
+
+  alias Explorer.Chain
+
+  @table :addresses_with_balance_counter
+
+  @cache_key "addresses_with_balance"
+
+  def table_name do
+    @table
+  end
+
+  def cache_key do
+    @cache_key
+  end
+
+  # It is undesirable to automatically start the consolidation in all environments.
+  # Consider the test environment: if the consolidation initiates but does not
+  # finish before a test ends, that test will fail. This way, hundreds of
+  # tests were failing before disabling the consolidation and the scheduler in
+  # the test env.
+  config = Application.get_env(:explorer, Explorer.Counters.AddessesWithBalanceCounter)
+  @enable_consolidation Keyword.get(config, :enable_consolidation)
+
+  @doc """
+  Starts a process to periodically update the counter of the token holders.
+  """
+  @spec start_link(term()) :: GenServer.on_start()
+  def start_link(_) do
+    GenServer.start_link(__MODULE__, :ok, name: __MODULE__)
+  end
+
+  @impl true
+  def init(args) do
+    create_table()
+
+    if enable_consolidation?() do
+      Task.start_link(&consolidate/0)
+      schedule_next_consolidation()
+    end
+
+    {:ok, args}
+  end
+
+  def create_table do
+    opts = [
+      :set,
+      :named_table,
+      :public,
+      read_concurrency: true
+    ]
+
+    :ets.new(table_name(), opts)
+  end
+
+  defp schedule_next_consolidation do
+    if enable_consolidation?() do
+      Process.send_after(self(), :consolidate, :timer.minutes(30))
+    end
+  end
+
+  @doc """
+  Inserts new items into the `:ets` table.
+  """
+  def insert_counter({key, info}) do
+    :ets.insert(table_name(), {key, info})
+  end
+
+  @impl true
+  def handle_info(:consolidate, state) do
+    consolidate()
+
+    schedule_next_consolidation()
+
+    {:noreply, state}
+  end
+
+  @doc """
+  Fetches the info for a specific item from the `:ets` table.
+  """
+  def fetch do
+    do_fetch(:ets.lookup(table_name(), cache_key()))
+  end
+
+  defp do_fetch([{_, result}]), do: result
+  defp do_fetch([]), do: 0
+
+  @doc """
+  Consolidates the info by populating the `:ets` table with the current database information.
+  """
+  def consolidate do
+    counter = Chain.count_addresses_with_balance()
+
+    insert_counter({cache_key(), counter})
+  end
+
+  @doc """
+  Returns a boolean that indicates whether consolidation is enabled
+
+  In order to choose whether or not to enable the scheduler and the initial
+  consolidation, change the following Explorer config:
+
+  `config :explorer, Explorer.Counters.AddressesWithBalanceCounter, enable_consolidation: true`
+
+  to:
+
+  `config :explorer, Explorer.Counters.AddressesWithBalanceCounter, enable_consolidation: false`
+  """
+  def enable_consolidation?, do: @enable_consolidation
+end

apps/explorer/lib/explorer/counters/block_validation_counter.ex

@@ -15,6 +15,14 @@ defmodule Explorer.Counters.BlockValidationCounter do
     @table
   end
 
+  # It is undesirable to automatically start the consolidation in all environments.
+  # Consider the test environment: if the consolidation initiates but does not
+  # finish before a test ends, that test will fail. This way, hundreds of
+  # tests were failing before disabling the consolidation and the scheduler in
+  # the test env.
+  config = Application.get_env(:explorer, Explorer.Counters.BlockValidationCounter)
+  @enable_consolidation Keyword.get(config, :enable_consolidation)
+
   @doc """
   Creates a process to continually monitor the validation counts.
   """
@@ -28,7 +36,9 @@ defmodule Explorer.Counters.BlockValidationCounter do
   def init(args) do
     create_table()
 
+    if enable_consolidation?() do
       Task.start_link(&consolidate_blocks/0)
+    end
 
     Chain.subscribe_to_events(:blocks)
 
@@ -40,8 +50,7 @@ defmodule Explorer.Counters.BlockValidationCounter do
       :set,
       :named_table,
       :public,
-      read_concurrency: true,
+      read_concurrency: true
-      write_concurrency: true
     ]
 
     :ets.new(table_name(), opts)
@@ -91,4 +100,18 @@ defmodule Explorer.Counters.BlockValidationCounter do
 
     :ets.update_counter(table_name(), string_addr, number, default)
   end
+
+  @doc """
+  Returns a boolean that indicates whether consolidation is enabled
+
+  In order to choose whether or not to enable the scheduler and the initial
+  consolidation, change the following Explorer config:
+
+  `config :explorer, Explorer.Counters.BlockValidationCounter, enable_consolidation: true`
+
+  to:
+
+  `config :explorer, Explorer.Counters.BlockValidationCounter, enable_consolidation: false`
+  """
+  def enable_consolidation?, do: @enable_consolidation
 end

apps/explorer/test/explorer/chain/address_test.exs

@@ -2,6 +2,7 @@ defmodule Explorer.Chain.AddressTest do
   use Explorer.DataCase
 
   alias Explorer.Chain.Address
+  alias Explorer.Repo
 
   describe "changeset/2" do
     test "with valid attributes" do
@@ -15,4 +16,14 @@ defmodule Explorer.Chain.AddressTest do
       refute changeset.valid?
     end
   end
+
+  describe "count_with_fetched_coin_balance/0" do
+    test "returns the number of addresses with fetched_coin_balance greater than 0" do
+      insert(:address, fetched_coin_balance: 0)
+      insert(:address, fetched_coin_balance: 1)
+      insert(:address, fetched_coin_balance: 2)
+
+      assert Repo.one(Address.count_with_fetched_coin_balance()) == 2
+    end
+  end
 end

apps/explorer/test/explorer/chain_test.exs

@@ -24,13 +24,22 @@ defmodule Explorer.ChainTest do
 
   alias Explorer.Chain.Supply.ProofOfAuthority
 
-  alias Explorer.Counters.TokenHoldersCounter
+  alias Explorer.Counters.{AddessesWithBalanceCounter, TokenHoldersCounter}
 
   doctest Explorer.Chain
 
-  describe "address_estimated_count/1" do
+  describe "count_addresses_with_balance_from_cache/0" do
-    test "returns integer" do
+    test "returns the number of addresses with fetched_coin_balance > 0" do
-      assert is_integer(Chain.address_estimated_count())
+      insert(:address, fetched_coin_balance: 0)
+      insert(:address, fetched_coin_balance: 1)
+      insert(:address, fetched_coin_balance: 2)
+
+      AddessesWithBalanceCounter.consolidate()
+
+      addresses_with_balance = Chain.count_addresses_with_balance_from_cache()
+
+      assert is_integer(addresses_with_balance)
+      assert addresses_with_balance == 2
     end
   end
 

apps/explorer/test/explorer/counters/addresses_with_balance_counter_test.exs

@@ -0,0 +1,15 @@
+defmodule Explorer.Counters.AddessesWithBalanceCounterTest do
+  use Explorer.DataCase
+
+  alias Explorer.Counters.AddessesWithBalanceCounter
+
+  test "populates the cache with the number of addresses with fetched coin balance greater than 0" do
+    insert(:address, fetched_coin_balance: 0)
+    insert(:address, fetched_coin_balance: 1)
+    insert(:address, fetched_coin_balance: 2)
+
+    AddessesWithBalanceCounter.consolidate()
+
+    assert AddessesWithBalanceCounter.fetch() == 2
+  end
+end