By using `COUNT(*)` instead of `COUNT(column)`, Postgres optimizes the count to use the row count recorded in the index. The ETS counter appeared necessary because inpull/1275/head95de1ebcb0, `COUNT(id)` was being used (95de1ebcb0/apps/explorer/lib/explorer/chain/token_transfer.ex (L138)), which seems reasonable in Ecto as you need some column to count, but in Postgres, it is more efficient to count everything with `*` as it implies "count rows". `COUNT(column)` requires Postgres check if any of the values are `NULL` and it does not optimize the query even if the column is `NOT NULL` in most cases like when the `id` was used. Testing on eth60-test database, `COUNT(*)` took 62ms while `COUNT(column)` (in this case `amount` since the table no longer has `id) took 16s demonstrating the vast speed improvement of `*` counting. ``` sql> SELECT COUNT(*) FROM token_transfers WHERE token_contract_address_hash = ( SELECT token_contract_address_hash FROM token_transfers LIMIT 1) [2018-12-21 07:51:02] 1 row retrieved starting from 1 in 70 ms (execution: 62 ms, fetching: 8 ms) sql> SELECT COUNT(amount) FROM token_transfers WHERE token_contract_address_hash = ( SELECT token_contract_address_hash FROM token_transfers LIMIT 1) [2018-12-21 07:51:43] 1 row retrieved starting from 1 in 16 s 862 ms (execution: 16 s 855 ms, fetching: 7 ms) ```
Luke Imhoff
6 years ago
parent
5999db7fab
commit
f67241cfc0
@ -1,114 +0,0 @@ |
||||
defmodule Explorer.Counters.TokenTransferCounter do |
||||
use GenServer |
||||
|
||||
@moduledoc """ |
||||
Module responsible for fetching and consolidating the number of transfers |
||||
from a token. |
||||
""" |
||||
|
||||
alias Explorer.Chain |
||||
alias Explorer.Chain.{Hash, TokenTransfer} |
||||
|
||||
@table :token_transfer_counter |
||||
|
||||
# It is undesirable to automatically start the consolidation in all environments. |
||||
# Consider the test environment: if the consolidation initiates but does not |
||||
# finish before a test ends, that test will fail. This way, hundreds of |
||||
# tests were failing before disabling the consolidation and the scheduler in |
||||
# the test env. |
||||
config = Application.get_env(:explorer, Explorer.Counters.TokenHoldersCounter) |
||||
@enable_consolidation Keyword.get(config, :enable_consolidation) |
||||
|
||||
@doc """ |
||||
Returns a boolean that indicates whether consolidation is enabled |
||||
|
||||
In order to choose whether or not to enable the initial consolidation, change the following Explorer config: |
||||
|
||||
`config :explorer, Explorer.Counters.TokenTransferCounter, enable_consolidation: true` |
||||
|
||||
to: |
||||
|
||||
`config :explorer, Explorer.Counters.TokenTransferCounter, enable_consolidation: false` |
||||
""" |
||||
def enable_consolidation?, do: @enable_consolidation |
||||
|
||||
def table_name do |
||||
@table |
||||
end |
||||
|
||||
@doc """ |
||||
Starts a process to continually monitor the token counters. |
||||
""" |
||||
@spec start_link(term()) :: GenServer.on_start() |
||||
def start_link(_) do |
||||
GenServer.start_link(__MODULE__, :ok, name: __MODULE__) |
||||
end |
||||
|
||||
## Server |
||||
@impl true |
||||
def init(args) do |
||||
create_table() |
||||
|
||||
if enable_consolidation?() do |
||||
Task.start_link(&consolidate/0) |
||||
end |
||||
|
||||
Chain.subscribe_to_events(:token_transfers) |
||||
|
||||
{:ok, args} |
||||
end |
||||
|
||||
def create_table do |
||||
opts = [ |
||||
:set, |
||||
:named_table, |
||||
:public, |
||||
read_concurrency: true, |
||||
write_concurrency: true |
||||
] |
||||
|
||||
:ets.new(table_name(), opts) |
||||
end |
||||
|
||||
@doc """ |
||||
Consolidates the number of token transfers grouped by token. |
||||
""" |
||||
def consolidate do |
||||
TokenTransfer.each_count(fn {token_hash, total} -> |
||||
insert_or_update_counter(token_hash, total) |
||||
end) |
||||
end |
||||
|
||||
@doc """ |
||||
Fetches the number of transfers related to a token hash. |
||||
""" |
||||
@spec fetch(Hash.t()) :: non_neg_integer |
||||
def fetch(token_hash) do |
||||
do_fetch(:ets.lookup(table_name(), to_string(token_hash))) |
||||
end |
||||
|
||||
defp do_fetch([{_, result} | _]), do: result |
||||
defp do_fetch([]), do: 0 |
||||
|
||||
@impl true |
||||
def handle_info({:chain_event, :token_transfers, _type, token_transfers}, state) do |
||||
token_transfers |
||||
|> Enum.map(& &1.token_contract_address_hash) |
||||
|> Enum.each(&insert_or_update_counter(&1, 1)) |
||||
|
||||
{:noreply, state} |
||||
end |
||||
|
||||
@doc """ |
||||
Inserts a new item into the `:ets` table. |
||||
|
||||
When the record exist, the counter will be incremented by one. When the |
||||
record does not exist, the counter will be inserted with a default value. |
||||
""" |
||||
@spec insert_or_update_counter(Hash.t(), non_neg_integer) :: term() |
||||
def insert_or_update_counter(token_hash, number) do |
||||
default = {to_string(token_hash), 0} |
||||
|
||||
:ets.update_counter(table_name(), to_string(token_hash), number, default) |
||||
end |
||||
end |
||||
@ -1,56 +0,0 @@ |
||||
defmodule Explorer.Counters.TokenTransferCounterTest do |
||||
use Explorer.DataCase |
||||
|
||||
alias Explorer.Counters.TokenTransferCounter |
||||
|
||||
setup do |
||||
start_supervised!(TokenTransferCounter) |
||||
|
||||
:ok |
||||
end |
||||
|
||||
describe "consolidate/0" do |
||||
test "loads the token's transfers consolidate info" do |
||||
token_contract_address = insert(:contract_address) |
||||
token = insert(:token, contract_address: token_contract_address) |
||||
|
||||
transaction = |
||||
:transaction |
||||
|> insert() |
||||
|> with_block() |
||||
|
||||
insert( |
||||
:token_transfer, |
||||
to_address: build(:address), |
||||
transaction: transaction, |
||||
token_contract_address: token_contract_address, |
||||
token: token |
||||
) |
||||
|
||||
insert( |
||||
:token_transfer, |
||||
to_address: build(:address), |
||||
transaction: transaction, |
||||
token_contract_address: token_contract_address, |
||||
token: token |
||||
) |
||||
|
||||
TokenTransferCounter.consolidate() |
||||
|
||||
assert TokenTransferCounter.fetch(token.contract_address_hash) == 2 |
||||
end |
||||
end |
||||
|
||||
describe "fetch/1" do |
||||
test "fetches the total token transfers by token contract address hash" do |
||||
token_contract_address = insert(:contract_address) |
||||
token = insert(:token, contract_address: token_contract_address) |
||||
|
||||
assert TokenTransferCounter.fetch(token.contract_address_hash) == 0 |
||||
|
||||
TokenTransferCounter.insert_or_update_counter(token.contract_address_hash, 15) |
||||
|
||||
assert TokenTransferCounter.fetch(token.contract_address_hash) == 15 |
||||
end |
||||
end |
||||
end |
||||
Loading…
Reference in new issue