Store counters in database

pull/6324/head
sl1depengwyn 2 years ago
parent e5a3911fb1
commit b255e5ef23
  1. 8
      apps/block_scout_web/test/block_scout_web/controllers/verified_contracts_controller_test.exs
  2. 8
      apps/explorer/config/config.exs
  3. 8
      apps/explorer/config/runtime/test.exs
  4. 8
      apps/explorer/lib/explorer/application.ex
  5. 20
      apps/explorer/lib/explorer/chain.ex
  6. 43
      apps/explorer/lib/explorer/chain/cache/contracts_counter.ex
  7. 44
      apps/explorer/lib/explorer/chain/cache/new_contracts_counter.ex
  8. 44
      apps/explorer/lib/explorer/chain/cache/new_verified_contracts_counter.ex
  9. 43
      apps/explorer/lib/explorer/chain/cache/verified_contracts_counter.ex
  10. 6
      apps/explorer/test/explorer/chain/cache/contracts_counter_test.exs
  11. 6
      apps/explorer/test/explorer/chain/cache/new_contracts_counter_test.exs
  12. 6
      apps/explorer/test/explorer/chain/cache/new_verified_contracts_counter_test.exs
  13. 7
      apps/explorer/test/explorer/chain/cache/verified_contracts_counter_test.exs

@ -4,7 +4,13 @@ defmodule BlockScoutWeb.VerifiedContractsControllerTest do
import BlockScoutWeb.WebRouter.Helpers, only: [verified_contracts_path: 2, verified_contracts_path: 3]
alias Explorer.Chain.SmartContract
alias Explorer.Counters.{ContractsCounter, NewContractsCounter, NewVerifiedContractsCounter, VerifiedContractsCounter}
alias Explorer.Chain.Cache.{
ContractsCounter,
NewContractsCounter,
NewVerifiedContractsCounter,
VerifiedContractsCounter
}
describe "GET index/2" do
test "returns 200", %{conn: conn} do

@ -54,22 +54,22 @@ config :explorer, Explorer.Counters.AddressTokenUsdSum,
enabled: true,
enable_consolidation: true
config :explorer, Explorer.Counters.ContractsCounter,
config :explorer, Explorer.Chain.Cache.ContractsCounter,
enabled: true,
enable_consolidation: true,
update_interval_in_seconds: 30 * 60
config :explorer, Explorer.Counters.NewContractsCounter,
config :explorer, Explorer.Chain.Cache.NewContractsCounter,
enabled: true,
enable_consolidation: true,
update_interval_in_seconds: 30 * 60
config :explorer, Explorer.Counters.VerifiedContractsCounter,
config :explorer, Explorer.Chain.Cache.VerifiedContractsCounter,
enabled: true,
enable_consolidation: true,
update_interval_in_seconds: 30 * 60
config :explorer, Explorer.Counters.NewVerifiedContractsCounter,
config :explorer, Explorer.Chain.Cache.NewVerifiedContractsCounter,
enabled: true,
enable_consolidation: true,
update_interval_in_seconds: 30 * 60

@ -18,10 +18,10 @@ config :explorer, Explorer.Chain.Transaction.History.Historian, enabled: false
config :explorer, Explorer.Market.History.Historian, enabled: false
config :explorer, Explorer.Counters.AddressesCounter, enabled: false, enable_consolidation: false
config :explorer, Explorer.Counters.ContractsCounter, enabled: false, enable_consolidation: false
config :explorer, Explorer.Counters.NewContractsCounter, enabled: false, enable_consolidation: false
config :explorer, Explorer.Counters.VerifiedContractsCounter, enabled: false, enable_consolidation: false
config :explorer, Explorer.Counters.NewVerifiedContractsCounter, enabled: false, enable_consolidation: false
config :explorer, Explorer.Chain.Cache.ContractsCounter, enabled: false, enable_consolidation: false
config :explorer, Explorer.Chain.Cache.NewContractsCounter, enabled: false, enable_consolidation: false
config :explorer, Explorer.Chain.Cache.VerifiedContractsCounter, enabled: false, enable_consolidation: false
config :explorer, Explorer.Chain.Cache.NewVerifiedContractsCounter, enabled: false, enable_consolidation: false
config :explorer, Explorer.Market.History.Cataloger, enabled: false

@ -85,6 +85,10 @@ defmodule Explorer.Application do
configure(Explorer.KnownTokens),
configure(Explorer.Market.History.Cataloger),
configure(Explorer.Chain.Cache.TokenExchangeRate),
configure(Explorer.Chain.Cache.ContractsCounter),
configure(Explorer.Chain.Cache.NewContractsCounter),
configure(Explorer.Chain.Cache.VerifiedContractsCounter),
configure(Explorer.Chain.Cache.NewVerifiedContractsCounter),
configure(Explorer.Chain.Transaction.History.Historian),
configure(Explorer.Chain.Events.Listener),
configure(Explorer.Counters.AddressesWithBalanceCounter),
@ -97,10 +101,6 @@ defmodule Explorer.Application do
configure(Explorer.Counters.TokenTransfersCounter),
configure(Explorer.Counters.BlockBurnedFeeCounter),
configure(Explorer.Counters.BlockPriorityFeeCounter),
configure(Explorer.Counters.ContractsCounter),
configure(Explorer.Counters.NewContractsCounter),
configure(Explorer.Counters.VerifiedContractsCounter),
configure(Explorer.Counters.NewVerifiedContractsCounter),
configure(Explorer.Counters.AverageBlockTime),
configure(Explorer.Counters.Bridge),
configure(Explorer.Validator.MetadataProcessor),

@ -69,8 +69,12 @@ defmodule Explorer.Chain do
Accounts,
BlockNumber,
Blocks,
ContractsCounter,
NewContractsCounter,
NewVerifiedContractsCounter,
Transactions,
Uncles
Uncles,
VerifiedContractsCounter
}
alias Explorer.Chain.Import.Runner
@ -78,11 +82,7 @@ defmodule Explorer.Chain do
alias Explorer.Counters.{
AddressesCounter,
AddressesWithBalanceCounter,
ContractsCounter,
NewContractsCounter,
NewVerifiedContractsCounter,
VerifiedContractsCounter
AddressesWithBalanceCounter
}
alias Explorer.Market.MarketHistoryCache
@ -6545,7 +6545,7 @@ defmodule Explorer.Chain do
defp filter_contracts(basic_query, _), do: basic_query
def count_verified_contracts do
Repo.aggregate(SmartContract, :count)
Repo.aggregate(SmartContract, :count, timeout: :infinity)
end
def count_new_verified_contracts do
@ -6556,7 +6556,7 @@ defmodule Explorer.Chain do
)
query
|> Repo.aggregate(:count)
|> Repo.aggregate(:count, timeout: :infinity)
end
def count_contracts do
@ -6567,7 +6567,7 @@ defmodule Explorer.Chain do
)
query
|> Repo.aggregate(:count)
|> Repo.aggregate(:count, timeout: :infinity)
end
def count_new_contracts do
@ -6580,7 +6580,7 @@ defmodule Explorer.Chain do
)
query
|> Repo.aggregate(:count)
|> Repo.aggregate(:count, timeout: :infinity)
end
def count_verified_contracts_from_cache do

@ -1,6 +1,6 @@
defmodule Explorer.Counters.ContractsCounter do
defmodule Explorer.Chain.Cache.ContractsCounter do
@moduledoc """
Caches the number of contracts, new and verified.
Caches the number of contracts.
It loads the count asynchronously and in a time interval of 30 minutes.
"""
@ -8,24 +8,21 @@ defmodule Explorer.Counters.ContractsCounter do
use GenServer
alias Explorer.Chain
alias Explorer.Counters.Helper
@table :contracts_counter
@cache_key "all"
@counter_type "contracts_counter"
# It is undesirable to automatically start the consolidation in all environments.
# Consider the test environment: if the consolidation initiates but does not
# finish before a test ends, that test will fail. This way, hundreds of
# tests were failing before disabling the consolidation and the scheduler in
# the test env.
config = Application.compile_env(:explorer, Explorer.Counters.ContractsCounter)
config = Application.compile_env(:explorer, Explorer.Chain.Cache.ContractsCounter)
@enable_consolidation Keyword.get(config, :enable_consolidation)
@update_interval_in_seconds Keyword.get(config, :update_interval_in_seconds)
@doc """
Starts a process to periodically update the counter of the .
Starts a process to periodically update the counter of all the contracts.
"""
@spec start_link(term()) :: GenServer.on_start()
def start_link(_) do
@ -34,26 +31,13 @@ defmodule Explorer.Counters.ContractsCounter do
@impl true
def init(_args) do
create_table()
{:ok, %{consolidate?: enable_consolidation?()}, {:continue, :ok}}
end
def create_table do
Helper.create_cache_table(@table)
end
defp schedule_next_consolidation do
Process.send_after(self(), :consolidate, :timer.seconds(@update_interval_in_seconds))
end
@doc """
Inserts new items into the `:ets` table.
"""
def insert_counter({key, info}) do
:ets.insert(@table, {key, info})
end
@impl true
def handle_continue(:ok, %{consolidate?: true} = state) do
consolidate()
@ -76,19 +60,24 @@ defmodule Explorer.Counters.ContractsCounter do
end
@doc """
Fetches the info for a specific item from the `:ets` table.
Fetches the value for a `#{@counter_type}` counter type from the `last_fetched_counters` table.
"""
def fetch do
Helper.fetch_from_cache(@cache_key, @table)
Chain.get_last_fetched_counter(@counter_type)
end
@doc """
Consolidates the info by populating the `:ets` table with the current database information.
Consolidates the info by populating the `last_fetched_counters` table with the current database information.
"""
def consolidate do
all_counter = Chain.count_contracts()
insert_counter({@cache_key, all_counter})
params = %{
counter_type: @counter_type,
value: all_counter
}
Chain.upsert_last_fetched_counter(params)
end
@doc """
@ -97,11 +86,11 @@ defmodule Explorer.Counters.ContractsCounter do
In order to choose whether or not to enable the scheduler and the initial
consolidation, change the following Explorer config:
`config :explorer, Explorer.Counters.ContractsCounter, enable_consolidation: true`
`config :explorer, Explorer.Chain.Cache.ContractsCounter, enable_consolidation: true`
to:
`config :explorer, Explorer.Counters.ContractsCounter, enable_consolidation: false`
`config :explorer, Explorer.Chain.Cache.ContractsCounter, enable_consolidation: false`
"""
def enable_consolidation?, do: @enable_consolidation
end

@ -1,6 +1,6 @@
defmodule Explorer.Counters.NewContractsCounter do
defmodule Explorer.Chain.Cache.NewContractsCounter do
@moduledoc """
Caches the number of contracts, new and verified.
Caches the number of new contracts (created in last 24 hours).
It loads the count asynchronously and in a time interval of 30 minutes.
"""
@ -8,24 +8,22 @@ defmodule Explorer.Counters.NewContractsCounter do
use GenServer
alias Explorer.Chain
alias Explorer.Counters.Helper
@table :new_contracts_counter
@cache_key "new"
@counter_type "new_contracts_counter"
# It is undesirable to automatically start the consolidation in all environments.
# Consider the test environment: if the consolidation initiates but does not
# finish before a test ends, that test will fail. This way, hundreds of
# tests were failing before disabling the consolidation and the scheduler in
# the test env.
config = Application.compile_env(:explorer, Explorer.Counters.NewContractsCounter)
config = Application.compile_env(:explorer, Explorer.Chain.Cache.NewContractsCounter)
@enable_consolidation Keyword.get(config, :enable_consolidation)
@update_interval_in_seconds Keyword.get(config, :update_interval_in_seconds)
@doc """
Starts a process to periodically update the counter of the .
Starts a process to periodically update the counter of new
contracts. (created in last 24 hours)
"""
@spec start_link(term()) :: GenServer.on_start()
def start_link(_) do
@ -34,26 +32,13 @@ defmodule Explorer.Counters.NewContractsCounter do
@impl true
def init(_args) do
create_table()
{:ok, %{consolidate?: enable_consolidation?()}, {:continue, :ok}}
end
def create_table do
Helper.create_cache_table(@table)
end
defp schedule_next_consolidation do
Process.send_after(self(), :consolidate, :timer.seconds(@update_interval_in_seconds))
end
@doc """
Inserts new items into the `:ets` table.
"""
def insert_counter({key, info}) do
:ets.insert(@table, {key, info})
end
@impl true
def handle_continue(:ok, %{consolidate?: true} = state) do
consolidate()
@ -76,19 +61,24 @@ defmodule Explorer.Counters.NewContractsCounter do
end
@doc """
Fetches the info for a specific item from the `:ets` table.
Fetches the value for a `#{@counter_type}` counter type from the `last_fetched_counters` table.
"""
def fetch do
Helper.fetch_from_cache(@cache_key, @table)
Chain.get_last_fetched_counter(@counter_type)
end
@doc """
Consolidates the info by populating the `:ets` table with the current database information.
Consolidates the info by populating the `last_fetched_counters` table with the current database information.
"""
def consolidate do
new_all_counter = Chain.count_new_contracts()
insert_counter({@cache_key, new_all_counter})
params = %{
counter_type: @counter_type,
value: new_all_counter
}
Chain.upsert_last_fetched_counter(params)
end
@doc """
@ -97,11 +87,11 @@ defmodule Explorer.Counters.NewContractsCounter do
In order to choose whether or not to enable the scheduler and the initial
consolidation, change the following Explorer config:
`config :explorer, Explorer.Counters.NewContractsCounter, enable_consolidation: true`
`config :explorer, Explorer.Chain.Cache.NewContractsCounter, enable_consolidation: true`
to:
`config :explorer, Explorer.Counters.NewContractsCounter, enable_consolidation: false`
`config :explorer, Explorer.Chain.Cache.NewContractsCounter, enable_consolidation: false`
"""
def enable_consolidation?, do: @enable_consolidation
end

@ -1,6 +1,6 @@
defmodule Explorer.Counters.NewVerifiedContractsCounter do
defmodule Explorer.Chain.Cache.NewVerifiedContractsCounter do
@moduledoc """
Caches the number of contracts, new and verified.
Caches the number of new verfied contracts (verified in last 24 hours).
It loads the count asynchronously and in a time interval of 30 minutes.
"""
@ -8,24 +8,22 @@ defmodule Explorer.Counters.NewVerifiedContractsCounter do
use GenServer
alias Explorer.Chain
alias Explorer.Counters.Helper
@table :new_verified_contracts_counter
@cache_key "new_verified"
@counter_type "new_verified_contracts_counter"
# It is undesirable to automatically start the consolidation in all environments.
# Consider the test environment: if the consolidation initiates but does not
# finish before a test ends, that test will fail. This way, hundreds of
# tests were failing before disabling the consolidation and the scheduler in
# the test env.
config = Application.compile_env(:explorer, Explorer.Counters.NewVerifiedContractsCounter)
config = Application.compile_env(:explorer, Explorer.Chain.Cache.NewVerifiedContractsCounter)
@enable_consolidation Keyword.get(config, :enable_consolidation)
@update_interval_in_seconds Keyword.get(config, :update_interval_in_seconds)
@doc """
Starts a process to periodically update the counter of the .
Starts a process to periodically update the counter of new verified
contracts (verified in last 24 hours).
"""
@spec start_link(term()) :: GenServer.on_start()
def start_link(_) do
@ -34,26 +32,13 @@ defmodule Explorer.Counters.NewVerifiedContractsCounter do
@impl true
def init(_args) do
create_table()
{:ok, %{consolidate?: enable_consolidation?()}, {:continue, :ok}}
end
def create_table do
Helper.create_cache_table(@table)
end
defp schedule_next_consolidation do
Process.send_after(self(), :consolidate, :timer.seconds(@update_interval_in_seconds))
end
@doc """
Inserts new items into the `:ets` table.
"""
def insert_counter({key, info}) do
:ets.insert(@table, {key, info})
end
@impl true
def handle_continue(:ok, %{consolidate?: true} = state) do
consolidate()
@ -76,19 +61,24 @@ defmodule Explorer.Counters.NewVerifiedContractsCounter do
end
@doc """
Fetches the info for a specific item from the `:ets` table.
Fetches the value for a `#{@counter_type}` counter type from the `last_fetched_counters` table.
"""
def fetch do
Helper.fetch_from_cache(@cache_key, @table)
Chain.get_last_fetched_counter(@counter_type)
end
@doc """
Consolidates the info by populating the `:ets` table with the current database information.
Consolidates the info by populating the `last_fetched_counters` table with the current database information.
"""
def consolidate do
new_verified_counter = Chain.count_new_verified_contracts()
insert_counter({@cache_key, new_verified_counter})
params = %{
counter_type: @counter_type,
value: new_verified_counter
}
Chain.upsert_last_fetched_counter(params)
end
@doc """
@ -97,11 +87,11 @@ defmodule Explorer.Counters.NewVerifiedContractsCounter do
In order to choose whether or not to enable the scheduler and the initial
consolidation, change the following Explorer config:
`config :explorer, Explorer.Counters.NewVerifiedContractsCounter, enable_consolidation: true`
`config :explorer, Explorer.Chain.Cache.NewVerifiedContractsCounter, enable_consolidation: true`
to:
`config :explorer, Explorer.Counters.NewVerifiedContractsCounter, enable_consolidation: false`
`config :explorer, Explorer.Chain.Cache.NewVerifiedContractsCounter, enable_consolidation: false`
"""
def enable_consolidation?, do: @enable_consolidation
end

@ -1,6 +1,6 @@
defmodule Explorer.Counters.VerifiedContractsCounter do
defmodule Explorer.Chain.Cache.VerifiedContractsCounter do
@moduledoc """
Caches the number of contracts, new and verified.
Caches the number of verified contracts.
It loads the count asynchronously and in a time interval of 30 minutes.
"""
@ -8,24 +8,21 @@ defmodule Explorer.Counters.VerifiedContractsCounter do
use GenServer
alias Explorer.Chain
alias Explorer.Counters.Helper
@table :verified_contracts_counter
@cache_key "verified"
@counter_type "verified_contracts_counter"
# It is undesirable to automatically start the consolidation in all environments.
# Consider the test environment: if the consolidation initiates but does not
# finish before a test ends, that test will fail. This way, hundreds of
# tests were failing before disabling the consolidation and the scheduler in
# the test env.
config = Application.compile_env(:explorer, Explorer.Counters.VerifiedContractsCounter)
config = Application.compile_env(:explorer, Explorer.Chain.Cache.VerifiedContractsCounter)
@enable_consolidation Keyword.get(config, :enable_consolidation)
@update_interval_in_seconds Keyword.get(config, :update_interval_in_seconds)
@doc """
Starts a process to periodically update the counter of the .
Starts a process to periodically update the counter of verified contracts.
"""
@spec start_link(term()) :: GenServer.on_start()
def start_link(_) do
@ -34,26 +31,13 @@ defmodule Explorer.Counters.VerifiedContractsCounter do
@impl true
def init(_args) do
create_table()
{:ok, %{consolidate?: enable_consolidation?()}, {:continue, :ok}}
end
def create_table do
Helper.create_cache_table(@table)
end
defp schedule_next_consolidation do
Process.send_after(self(), :consolidate, :timer.seconds(@update_interval_in_seconds))
end
@doc """
Inserts new items into the `:ets` table.
"""
def insert_counter({key, info}) do
:ets.insert(@table, {key, info})
end
@impl true
def handle_continue(:ok, %{consolidate?: true} = state) do
consolidate()
@ -76,19 +60,24 @@ defmodule Explorer.Counters.VerifiedContractsCounter do
end
@doc """
Fetches the info for a specific item from the `:ets` table.
Fetches the value for a `#{@counter_type}` counter type from the `last_fetched_counters` table.
"""
def fetch do
Helper.fetch_from_cache(@cache_key, @table)
Chain.get_last_fetched_counter(@counter_type)
end
@doc """
Consolidates the info by populating the `:ets` table with the current database information.
Consolidates the info by populating the `last_fetched_counters` table with the current database information.
"""
def consolidate do
verified_counter = Chain.count_verified_contracts()
insert_counter({@cache_key, verified_counter})
params = %{
counter_type: @counter_type,
value: verified_counter
}
Chain.upsert_last_fetched_counter(params)
end
@doc """
@ -97,11 +86,11 @@ defmodule Explorer.Counters.VerifiedContractsCounter do
In order to choose whether or not to enable the scheduler and the initial
consolidation, change the following Explorer config:
`config :explorer, Explorer.Counters.VerifiedContractsCounter, enable_consolidation: true`
`config :explorer, Explorer.Chain.Cache.VerifiedContractsCounter, enable_consolidation: true`
to:
`config :explorer, Explorer.Counters.VerifiedContractsCounter, enable_consolidation: false`
`config :explorer, Explorer.Chain.Cache.VerifiedContractsCounter, enable_consolidation: false`
"""
def enable_consolidation?, do: @enable_consolidation
end

@ -1,7 +1,7 @@
defmodule Explorer.Counters.ContractsCounterTest do
defmodule Explorer.Chain.Cache.ContractsCounterTest do
use Explorer.DataCase
alias Explorer.Counters.ContractsCounter
alias Explorer.Chain.Cache.ContractsCounter
alias Explorer.Chain
test "populates the cache with the number of all contracts" do
@ -13,6 +13,6 @@ defmodule Explorer.Counters.ContractsCounterTest do
start_supervised!(ContractsCounter)
ContractsCounter.consolidate()
assert Chain.count_contracts_from_cache() == 4
assert Chain.count_contracts_from_cache() == Decimal.new(4)
end
end

@ -1,7 +1,7 @@
defmodule Explorer.Counters.NewContractsCounterTest do
defmodule Explorer.Chain.Cache.NewContractsCounterTest do
use Explorer.DataCase
alias Explorer.Counters.NewContractsCounter
alias Explorer.Chain.Cache.NewContractsCounter
alias Explorer.Chain
test "populates the cache with the number of new contracts (last 24h)" do
@ -24,6 +24,6 @@ defmodule Explorer.Counters.NewContractsCounterTest do
start_supervised!(NewContractsCounter)
NewContractsCounter.consolidate()
assert Chain.count_new_contracts_from_cache() == 2
assert Chain.count_new_contracts_from_cache() == Decimal.new(2)
end
end

@ -1,7 +1,7 @@
defmodule Explorer.Counters.NewVerifiedContractsCounterTest do
defmodule Explorer.Chain.Cache.NewVerifiedContractsCounterTest do
use Explorer.DataCase
alias Explorer.Counters.NewVerifiedContractsCounter
alias Explorer.Chain.Cache.NewVerifiedContractsCounter
alias Explorer.Chain
test "populates the cache with the number of new verified contracts (last 24h)" do
@ -13,6 +13,6 @@ defmodule Explorer.Counters.NewVerifiedContractsCounterTest do
start_supervised!(NewVerifiedContractsCounter)
NewVerifiedContractsCounter.consolidate()
assert Chain.count_new_verified_contracts_from_cache() == 2
assert Chain.count_new_verified_contracts_from_cache() == Decimal.new(2)
end
end

@ -1,9 +1,8 @@
defmodule Explorer.Counters.VerifiedContractsCounterTest do
defmodule Explorer.Chain.Cache.VerifiedContractsCounterTest do
use Explorer.DataCase
alias Explorer.Counters.VerifiedContractsCounter
alias Explorer.Chain.Cache.VerifiedContractsCounter
alias Explorer.Chain
alias Explorer.Chain.Transaction
test "populates the cache with the number of verified contracts" do
insert(:smart_contract)
@ -13,6 +12,6 @@ defmodule Explorer.Counters.VerifiedContractsCounterTest do
start_supervised!(VerifiedContractsCounter)
VerifiedContractsCounter.consolidate()
assert Chain.count_verified_contracts_from_cache() == 3
assert Chain.count_verified_contracts_from_cache() == Decimal.new(3)
end
end
Loading…
Cancel
Save