From 74512a094f94227ab6380a2a0dc06ffda689540b Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Mon, 5 Oct 2020 17:33:42 +0300 Subject: [PATCH 1/6] Caching of address transactions counter, remove query limit --- CHANGELOG.md | 1 + .../assets/js/pages/address.js | 2 +- .../controllers/address_controller.ex | 19 +-- .../controllers/address_controller_test.exs | 4 +- apps/explorer/config/config.exs | 11 ++ apps/explorer/lib/explorer/application.ex | 1 + apps/explorer/lib/explorer/chain.ex | 31 +++-- .../counters/address_transactions_counter.ex | 114 ++++++++++++++++++ 8 files changed, 156 insertions(+), 27 deletions(-) create mode 100644 apps/explorer/lib/explorer/counters/address_transactions_counter.ex diff --git a/CHANGELOG.md b/CHANGELOG.md index 02da3ead56..87c8ba29ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## Current ### Features +- [#3330](https://github.com/poanetwork/blockscout/pull/3330) - Caching of address transactions counter, remove query 10_000 rows limit ### Fixes diff --git a/apps/block_scout_web/assets/js/pages/address.js b/apps/block_scout_web/assets/js/pages/address.js index eab3cf0749..0c86066f71 100644 --- a/apps/block_scout_web/assets/js/pages/address.js +++ b/apps/block_scout_web/assets/js/pages/address.js @@ -94,7 +94,7 @@ const elements = { render ($el, state, oldState) { if (state.countersFetched && state.transactionCount) { if (oldState.transactionCount === state.transactionCount) return - $el.empty().append('>= ' + numeral(state.transactionCount).format() + ' Transactions') + $el.empty().append(numeral(state.transactionCount).format() + ' Transactions') $el.show() $el.parent('.address-detail-item').removeAttr('style') } else { diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/address_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/address_controller.ex index 62587223ac..dd888e0b14 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/address_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/address_controller.ex @@ -4,6 +4,7 @@ defmodule BlockScoutWeb.AddressController do import BlockScoutWeb.Chain, only: [paging_options: 1, next_page_params: 3, split_list_by_page: 1] alias BlockScoutWeb.{AccessHelpers, AddressView} + alias Explorer.Counters.AddressTransactionsCounter alias Explorer.{Chain, Market} alias Explorer.ExchangeRates.Token alias Phoenix.View @@ -109,25 +110,11 @@ defmodule BlockScoutWeb.AddressController do |> List.to_tuple() end - defp transaction_count(address) do - if contract?(address) do - incoming_transaction_count = Chain.address_to_incoming_transaction_count(address.hash) - - if incoming_transaction_count == 0 do - Chain.total_transactions_sent_by_address(address.hash) - else - incoming_transaction_count - end - else - Chain.total_transactions_sent_by_address(address.hash) - end + def transaction_count(address) do + AddressTransactionsCounter.fetch(address) end defp validation_count(address) do Chain.address_to_validation_count(address.hash) end - - defp contract?(%{contract_code: nil}), do: false - - defp contract?(%{contract_code: _}), do: true end diff --git a/apps/block_scout_web/test/block_scout_web/controllers/address_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/address_controller_test.exs index 32a253b644..d68f67f25c 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/address_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/address_controller_test.exs @@ -6,7 +6,7 @@ defmodule BlockScoutWeb.AddressControllerTest do import Mox alias Explorer.Chain.Address - alias Explorer.Counters.AddressesCounter + alias Explorer.Counters.{AddressesCounter, AddressTransactionsCounter} describe "GET index/2" do setup :set_mox_global @@ -85,7 +85,7 @@ defmodule BlockScoutWeb.AddressControllerTest do assert conn.status == 200 {:ok, response} = Jason.decode(conn.resp_body) - assert %{"transaction_count" => 0, "validation_count" => 0} == response + assert %{"transaction_count" => nil, "validation_count" => 0} == response end end end diff --git a/apps/explorer/config/config.exs b/apps/explorer/config/config.exs index 743eebf3cd..f61e4f7289 100644 --- a/apps/explorer/config/config.exs +++ b/apps/explorer/config/config.exs @@ -88,6 +88,17 @@ config :explorer, Explorer.Counters.AddressesCounter, enable_consolidation: true, update_interval_in_seconds: balances_update_interval || 30 * 60 +address_transactions_counter_cache_period = + case Integer.parse(System.get_env("ADDRESS_TRANSACTIONS_COUNTER_CACHE_PERIOD", "")) do + {secs, ""} -> :timer.seconds(secs) + _ -> :timer.hours(1) + end + +config :explorer, Explorer.Counters.AddressTransactionsCounter, + enabled: true, + enable_consolidation: true, + period: address_transactions_counter_cache_period + bridge_market_cap_update_interval = if System.get_env("BRIDGE_MARKET_CAP_UPDATE_INTERVAL") do case Integer.parse(System.get_env("BRIDGE_MARKET_CAP_UPDATE_INTERVAL")) do diff --git a/apps/explorer/lib/explorer/application.ex b/apps/explorer/lib/explorer/application.ex index 590dbf3a21..4d238df82b 100644 --- a/apps/explorer/lib/explorer/application.ex +++ b/apps/explorer/lib/explorer/application.ex @@ -78,6 +78,7 @@ defmodule Explorer.Application do configure(Explorer.Chain.Events.Listener), configure(Explorer.Counters.AddressesWithBalanceCounter), configure(Explorer.Counters.AddressesCounter), + configure(Explorer.Counters.AddressTransactionsCounter), configure(Explorer.Counters.AverageBlockTime), configure(Explorer.Counters.Bridge), configure(Explorer.Validator.MetadataProcessor), diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index 0f584ef753..47ce8f019e 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -714,15 +714,11 @@ defmodule Explorer.Chain do @spec address_to_incoming_transaction_count(Hash.Address.t()) :: non_neg_integer() def address_to_incoming_transaction_count(address_hash) do - paging_options = %PagingOptions{page_size: @max_incoming_transactions_count} - - base_query = - paging_options - |> fetch_transactions() - to_address_query = - base_query - |> where([t], t.to_address_hash == ^address_hash) + from( + transaction in Transaction, + where: transaction.to_address_hash == ^address_hash + ) Repo.aggregate(to_address_query, :count, :hash, timeout: :infinity) end @@ -1949,6 +1945,25 @@ defmodule Explorer.Chain do Repo.one(query) end + @spec address_to_transaction_count(Address.t()) :: non_neg_integer() + def address_to_transaction_count(address) do + if contract?(address) do + incoming_transaction_count = address_to_incoming_transaction_count(address.hash) + + if incoming_transaction_count == 0 do + total_transactions_sent_by_address(address.hash) + else + incoming_transaction_count + end + else + total_transactions_sent_by_address(address.hash) + end + end + + defp contract?(%{contract_code: nil}), do: false + + defp contract?(%{contract_code: _}), do: true + @doc """ Returns a stream of unfetched `t:Explorer.Chain.Address.CoinBalance.t/0`. diff --git a/apps/explorer/lib/explorer/counters/address_transactions_counter.ex b/apps/explorer/lib/explorer/counters/address_transactions_counter.ex new file mode 100644 index 0000000000..0973c81532 --- /dev/null +++ b/apps/explorer/lib/explorer/counters/address_transactions_counter.ex @@ -0,0 +1,114 @@ +defmodule Explorer.Counters.AddressTransactionsCounter do + @moduledoc """ + Caches Address transactions counter. + """ + use GenServer + + alias Explorer.Chain + + @cache_name :address_transactions_counter + @last_update_key "last_update" + @cache_period Application.get_env(:explorer, __MODULE__)[:period] + + @ets_opts [ + :set, + :named_table, + :public, + read_concurrency: true + ] + + config = Application.get_env(:explorer, Explorer.Counters.AddressesCounter) + @enable_consolidation Keyword.get(config, :enable_consolidation) + + @spec start_link(term()) :: GenServer.on_start() + def start_link(_) do + GenServer.start_link(__MODULE__, :ok, name: __MODULE__) + end + + @impl true + def init(_args) do + create_cache_table() + + {:ok, %{consolidate?: enable_consolidation?()}, {:continue, :ok}} + end + + @impl true + def handle_continue(:ok, %{consolidate?: true} = state) do + {:noreply, state} + end + + @impl true + def handle_continue(:ok, state) do + {:noreply, state} + end + + @impl true + def handle_info(:consolidate, state) do + {:noreply, state} + end + + def fetch(address) do + if cache_expired?() do + Task.async(fn -> + update_cache(address) + end) + end + + address_hash_string = get_address_hash_string(address) + fetch_from_cache("hash_#{address_hash_string}") + end + + def cache_name, do: @cache_name + + def updated_at_key, do: @last_update_key + + defp cache_expired? do + updated_at = fetch_from_cache(@last_update_key) + + cond do + is_nil(updated_at) -> true + current_time() - updated_at > @cache_period -> true + true -> false + end + end + + defp update_cache(address) do + put_into_cache(@last_update_key, current_time()) + + new_data = Chain.address_to_transaction_count(address) + address_hash_string = get_address_hash_string(address) + put_into_cache("hash_#{address_hash_string}", new_data) + end + + defp fetch_from_cache(key) do + case :ets.lookup(@cache_name, key) do + [{_, value}] -> + value + + [] -> + nil + end + end + + defp put_into_cache(key, value) do + :ets.insert(@cache_name, {key, value}) + end + + defp get_address_hash_string(address) do + Base.encode16(address.hash.bytes, case: :lower) + end + + defp current_time do + utc_now = DateTime.utc_now() + + DateTime.to_unix(utc_now, :millisecond) + end + + def create_cache_table do + if :ets.whereis(@cache_name) == :undefined do + :ets.new(@cache_name, @ets_opts) + end + end + + def enable_consolidation?, do: @enable_consolidation +end From 3dacb0f94b7eea1136bad391f7e8f37130bad35c Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Tue, 6 Oct 2020 09:02:21 +0300 Subject: [PATCH 2/6] Update last_update_key when new data received --- .../lib/explorer/counters/address_transactions_counter.ex | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/explorer/lib/explorer/counters/address_transactions_counter.ex b/apps/explorer/lib/explorer/counters/address_transactions_counter.ex index 0973c81532..8445c4a12a 100644 --- a/apps/explorer/lib/explorer/counters/address_transactions_counter.ex +++ b/apps/explorer/lib/explorer/counters/address_transactions_counter.ex @@ -73,10 +73,9 @@ defmodule Explorer.Counters.AddressTransactionsCounter do end defp update_cache(address) do - put_into_cache(@last_update_key, current_time()) - new_data = Chain.address_to_transaction_count(address) address_hash_string = get_address_hash_string(address) + put_into_cache(@last_update_key, current_time()) put_into_cache("hash_#{address_hash_string}", new_data) end From c25d276d8800c2d486c9165b5ef9a085c6115b9b Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Tue, 6 Oct 2020 09:31:58 +0300 Subject: [PATCH 3/6] Task.start_link instead of Task.async --- .../lib/explorer/counters/address_transactions_counter.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/explorer/lib/explorer/counters/address_transactions_counter.ex b/apps/explorer/lib/explorer/counters/address_transactions_counter.ex index 8445c4a12a..07ecc5e16b 100644 --- a/apps/explorer/lib/explorer/counters/address_transactions_counter.ex +++ b/apps/explorer/lib/explorer/counters/address_transactions_counter.ex @@ -49,7 +49,7 @@ defmodule Explorer.Counters.AddressTransactionsCounter do def fetch(address) do if cache_expired?() do - Task.async(fn -> + Task.start_link(fn -> update_cache(address) end) end From a7f94afb1a72979383ab7b58eae89d47b4bb6644 Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Tue, 6 Oct 2020 10:43:11 +0300 Subject: [PATCH 4/6] Fix last_update cache key to store txs count --- .../controllers/address_controller_test.exs | 2 +- .../counters/address_transactions_counter.ex | 13 ++++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/apps/block_scout_web/test/block_scout_web/controllers/address_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/address_controller_test.exs index d68f67f25c..cf0126937d 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/address_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/address_controller_test.exs @@ -85,7 +85,7 @@ defmodule BlockScoutWeb.AddressControllerTest do assert conn.status == 200 {:ok, response} = Jason.decode(conn.resp_body) - assert %{"transaction_count" => nil, "validation_count" => 0} == response + assert %{"transaction_count" => 0, "validation_count" => 0} == response end end end diff --git a/apps/explorer/lib/explorer/counters/address_transactions_counter.ex b/apps/explorer/lib/explorer/counters/address_transactions_counter.ex index 07ecc5e16b..edcdf3de5b 100644 --- a/apps/explorer/lib/explorer/counters/address_transactions_counter.ex +++ b/apps/explorer/lib/explorer/counters/address_transactions_counter.ex @@ -48,7 +48,7 @@ defmodule Explorer.Counters.AddressTransactionsCounter do end def fetch(address) do - if cache_expired?() do + if cache_expired?(address) do Task.start_link(fn -> update_cache(address) end) @@ -60,10 +60,9 @@ defmodule Explorer.Counters.AddressTransactionsCounter do def cache_name, do: @cache_name - def updated_at_key, do: @last_update_key - - defp cache_expired? do - updated_at = fetch_from_cache(@last_update_key) + defp cache_expired?(address) do + address_hash_string = get_address_hash_string(address) + updated_at = fetch_from_cache("hash_#{address_hash_string}_#{@last_update_key}") cond do is_nil(updated_at) -> true @@ -75,7 +74,7 @@ defmodule Explorer.Counters.AddressTransactionsCounter do defp update_cache(address) do new_data = Chain.address_to_transaction_count(address) address_hash_string = get_address_hash_string(address) - put_into_cache(@last_update_key, current_time()) + put_into_cache("hash_#{address_hash_string}_#{@last_update_key}", current_time()) put_into_cache("hash_#{address_hash_string}", new_data) end @@ -85,7 +84,7 @@ defmodule Explorer.Counters.AddressTransactionsCounter do value [] -> - nil + 0 end end From 72007f0f34fe8116abdd306d41df9160b35406c2 Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Tue, 6 Oct 2020 11:09:56 +0300 Subject: [PATCH 5/6] Update address txs count cache: commit update date at the beginning --- .../lib/explorer/counters/address_transactions_counter.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/explorer/lib/explorer/counters/address_transactions_counter.ex b/apps/explorer/lib/explorer/counters/address_transactions_counter.ex index edcdf3de5b..3b14795855 100644 --- a/apps/explorer/lib/explorer/counters/address_transactions_counter.ex +++ b/apps/explorer/lib/explorer/counters/address_transactions_counter.ex @@ -72,9 +72,9 @@ defmodule Explorer.Counters.AddressTransactionsCounter do end defp update_cache(address) do + put_into_cache("hash_#{address_hash_string}_#{@last_update_key}", current_time()) new_data = Chain.address_to_transaction_count(address) address_hash_string = get_address_hash_string(address) - put_into_cache("hash_#{address_hash_string}_#{@last_update_key}", current_time()) put_into_cache("hash_#{address_hash_string}", new_data) end From ab9a7de02c943cfe242f45bd1e3bab0953ebe1e8 Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Tue, 6 Oct 2020 11:23:39 +0300 Subject: [PATCH 6/6] Update address txs count cache: commit update date at the beginning fix --- .../lib/explorer/counters/address_transactions_counter.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/explorer/lib/explorer/counters/address_transactions_counter.ex b/apps/explorer/lib/explorer/counters/address_transactions_counter.ex index 3b14795855..6a2313da45 100644 --- a/apps/explorer/lib/explorer/counters/address_transactions_counter.ex +++ b/apps/explorer/lib/explorer/counters/address_transactions_counter.ex @@ -72,9 +72,9 @@ defmodule Explorer.Counters.AddressTransactionsCounter do end defp update_cache(address) do + address_hash_string = get_address_hash_string(address) put_into_cache("hash_#{address_hash_string}_#{@last_update_key}", current_time()) new_data = Chain.address_to_transaction_count(address) - address_hash_string = get_address_hash_string(address) put_into_cache("hash_#{address_hash_string}", new_data) end