From 326b70fa5289fbdab484049b12ac33407d52e931 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 3 Sep 2019 11:48:28 +0300 Subject: [PATCH 01/15] set correct last calue for coin balances chart data Sometimes the last value for coin history chart data is incorrect because when selecting records they are grouped by day and max value is selected in that day. So if there are a couple of coin balance changes only max value is shown. This PR always shows the latest value from the database by block number. --- apps/explorer/lib/explorer/chain.ex | 7 +++++++ .../lib/explorer/chain/address/coin_balance.ex | 2 +- apps/explorer/test/explorer/chain_test.exs | 16 ++++++++++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index 1581bd4846..b2a7c6fdb1 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -3002,9 +3002,16 @@ defmodule Explorer.Chain do address_hash |> CoinBalance.balances_by_day(latest_block_timestamp) |> Repo.all() + |> replace_last_value(latest_block_timestamp) |> normalize_balances_by_day() end + defp replace_last_value(items, %{value: value, timestamp: timestamp}) do + List.replace_at(items, -1, %{date: Date.convert!(timestamp, Calendar.ISO), value: value}) + end + + defp replace_last_value(items, _), do: items + defp normalize_balances_by_day(balances_by_day) do result = balances_by_day diff --git a/apps/explorer/lib/explorer/chain/address/coin_balance.ex b/apps/explorer/lib/explorer/chain/address/coin_balance.ex index 537f6a0483..f3647a8f54 100644 --- a/apps/explorer/lib/explorer/chain/address/coin_balance.ex +++ b/apps/explorer/lib/explorer/chain/address/coin_balance.ex @@ -112,7 +112,7 @@ defmodule Explorer.Chain.Address.CoinBalance do |> join(:inner, [cb], b in Block, on: cb.block_number == b.number) |> where([cb], cb.address_hash == ^address_hash) |> last(:block_number) - |> select([cb, b], %{timestamp: b.timestamp}) + |> select([cb, b], %{timestamp: b.timestamp, value: cb.value}) end def changeset(%__MODULE__{} = balance, params) do diff --git a/apps/explorer/test/explorer/chain_test.exs b/apps/explorer/test/explorer/chain_test.exs index 187d503621..d9bfec58f4 100644 --- a/apps/explorer/test/explorer/chain_test.exs +++ b/apps/explorer/test/explorer/chain_test.exs @@ -3908,6 +3908,22 @@ defmodule Explorer.ChainTest do %{date: today |> NaiveDateTime.to_date() |> Date.to_string(), value: Decimal.new("1E-15")} ] end + + test "uses last block value if there a couple of change in the same day" do + address = insert(:address) + today = NaiveDateTime.utc_now() + past = Timex.shift(today, hours: -1) + + block_now = insert(:block, timestamp: today) + insert(:fetched_balance, address_hash: address.hash, value: 1, block_number: block_now.number) + + block_past = insert(:block, timestamp: past) + insert(:fetched_balance, address_hash: address.hash, value: 0, block_number: block_past.number) + + [balance] = Chain.address_to_balances_by_day(address.hash) + + assert balance.value == Decimal.new(0) + end end describe "block_combined_rewards/1" do From bf23cc0ec17d0e99bfbccd9c06cc3450a196a9f0 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 3 Sep 2019 11:54:28 +0300 Subject: [PATCH 02/15] add CHANGELOG entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca60257372..8ccd24aa31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - [#2497](https://github.com/poanetwork/blockscout/pull/2497) - Add generic Ordered Cache behaviour and implementation ### Fixes +- [#2660](https://github.com/poanetwork/blockscout/pull/2660) - set correct last value for coin balances chart data - [#2468](https://github.com/poanetwork/blockscout/pull/2468) - fix confirmations for non consensus blocks - [#2610](https://github.com/poanetwork/blockscout/pull/2610) - use CoinGecko instead of CoinMarketcap for exchange rates - [#2640](https://github.com/poanetwork/blockscout/pull/2640) - SVG network icons From acd7fb53b31200f6995a2bfad67f3c3a9286c05e Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 3 Sep 2019 11:58:11 +0300 Subject: [PATCH 03/15] fix test --- apps/explorer/test/explorer/chain_test.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/explorer/test/explorer/chain_test.exs b/apps/explorer/test/explorer/chain_test.exs index d9bfec58f4..d673336ef6 100644 --- a/apps/explorer/test/explorer/chain_test.exs +++ b/apps/explorer/test/explorer/chain_test.exs @@ -3914,10 +3914,10 @@ defmodule Explorer.ChainTest do today = NaiveDateTime.utc_now() past = Timex.shift(today, hours: -1) - block_now = insert(:block, timestamp: today) + block_now = insert(:block, timestamp: today, number: 1) insert(:fetched_balance, address_hash: address.hash, value: 1, block_number: block_now.number) - block_past = insert(:block, timestamp: past) + block_past = insert(:block, timestamp: past, number: 2) insert(:fetched_balance, address_hash: address.hash, value: 0, block_number: block_past.number) [balance] = Chain.address_to_balances_by_day(address.hash) From 05a00b4cc90eb981e2dd118e22abc5635e3ce8ce Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 3 Sep 2019 12:25:57 +0300 Subject: [PATCH 04/15] fix tests --- .../address_coin_balance_by_day_controller_test.exs | 4 ++-- apps/explorer/test/explorer/chain_test.exs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/block_scout_web/test/block_scout_web/controllers/address_coin_balance_by_day_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/address_coin_balance_by_day_controller_test.exs index 1496bb24e9..df714247c2 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/address_coin_balance_by_day_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/address_coin_balance_by_day_controller_test.exs @@ -5,8 +5,8 @@ defmodule BlockScoutWeb.AddressCoinBalanceByDayControllerTest do test "returns the coin balance history grouped by date", %{conn: conn} do address = insert(:address) noon = Timex.now() |> Timex.beginning_of_day() |> Timex.set(hour: 12) - block = insert(:block, timestamp: noon) - block_one_day_ago = insert(:block, timestamp: Timex.shift(noon, days: -1)) + block = insert(:block, timestamp: noon, number: 2) + block_one_day_ago = insert(:block, timestamp: Timex.shift(noon, days: -1), number: 1) insert(:fetched_balance, address_hash: address.hash, value: 1000, block_number: block.number) insert(:fetched_balance, address_hash: address.hash, value: 2000, block_number: block_one_day_ago.number) diff --git a/apps/explorer/test/explorer/chain_test.exs b/apps/explorer/test/explorer/chain_test.exs index d673336ef6..3ac60b6963 100644 --- a/apps/explorer/test/explorer/chain_test.exs +++ b/apps/explorer/test/explorer/chain_test.exs @@ -3879,9 +3879,9 @@ defmodule Explorer.ChainTest do address = insert(:address) today = NaiveDateTime.utc_now() noon = Timex.set(today, hour: 12) - block = insert(:block, timestamp: noon) + block = insert(:block, timestamp: noon, number: 50) yesterday = Timex.shift(noon, days: -1) - block_one_day_ago = insert(:block, timestamp: yesterday) + block_one_day_ago = insert(:block, timestamp: yesterday, number: 49) insert(:fetched_balance, address_hash: address.hash, value: 1000, block_number: block.number) insert(:fetched_balance, address_hash: address.hash, value: 2000, block_number: block_one_day_ago.number) From 4c8953a883576488e3ba0c104ac1c30e23919bb3 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Wed, 18 Sep 2019 15:13:36 +0300 Subject: [PATCH 05/15] fix empty total_supply in coin gecko response --- apps/explorer/lib/explorer/exchange_rates/source/coin_gecko.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/explorer/lib/explorer/exchange_rates/source/coin_gecko.ex b/apps/explorer/lib/explorer/exchange_rates/source/coin_gecko.ex index 9d4cfba3f4..b25cc53ff5 100644 --- a/apps/explorer/lib/explorer/exchange_rates/source/coin_gecko.ex +++ b/apps/explorer/lib/explorer/exchange_rates/source/coin_gecko.ex @@ -26,7 +26,7 @@ defmodule Explorer.ExchangeRates.Source.CoinGecko do [ %Token{ available_supply: to_decimal(market_data["circulating_supply"]), - total_supply: to_decimal(market_data["total_supply"]), + total_supply: to_decimal(market_data["total_supply"]) || to_decimal(market_data["circulating_supply"]), btc_value: btc_value, id: json_data["id"], last_updated: last_updated, From 66503e4164b0460b1b731171dd6c040d4b1ddbcd Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Wed, 18 Sep 2019 15:15:31 +0300 Subject: [PATCH 06/15] add CHANGELOG entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f3a0f91771..c818adeda8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - [#2663](https://github.com/poanetwork/blockscout/pull/2663) - Fetch address counters in parallel ### Fixes +- [#2706](https://github.com/poanetwork/blockscout/pull/2706) - fix empty total_supply in coin gecko response - [#2701](https://github.com/poanetwork/blockscout/pull/2701) - Exclude nonconsensus blocks from avg block time calculation by default - [#2696](https://github.com/poanetwork/blockscout/pull/2696) - do not update fetched_coin_balance with nil - [#2693](https://github.com/poanetwork/blockscout/pull/2693) - remove non consensus internal transactions From af7782f2fe46983159f9e8e00966e57cb1721e8f Mon Sep 17 00:00:00 2001 From: pasqu4le Date: Wed, 4 Sep 2019 18:29:57 +0200 Subject: [PATCH 07/15] Add ETS-based cache for accounts page --- CHANGELOG.md | 1 + .../block_scout_web/test/support/conn_case.ex | 2 + .../test/support/feature_case.ex | 2 + apps/explorer/config/config.exs | 4 + apps/explorer/lib/explorer/application.ex | 4 +- apps/explorer/lib/explorer/chain.ex | 31 ++++++++ .../lib/explorer/chain/cache/accounts.ex | 73 +++++++++++++++++++ .../explorer/chain/cache/accounts_test.exs | 42 +++++++++++ apps/explorer/test/support/data_case.ex | 2 + apps/indexer/lib/indexer/block/fetcher.ex | 5 +- .../lib/indexer/block/realtime/fetcher.ex | 3 + .../lib/indexer/fetcher/block_reward.ex | 5 +- .../lib/indexer/fetcher/coin_balance.ex | 5 +- .../indexer/fetcher/coin_balance_on_demand.ex | 8 +- .../lib/indexer/fetcher/contract_code.ex | 4 +- .../indexer/fetcher/internal_transaction.ex | 3 + .../indexer/fetcher/pending_transaction.ex | 4 +- .../lib/indexer/fetcher/uncle_block.ex | 4 +- 18 files changed, 193 insertions(+), 9 deletions(-) create mode 100644 apps/explorer/lib/explorer/chain/cache/accounts.ex create mode 100644 apps/explorer/test/explorer/chain/cache/accounts_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index f3a0f91771..53299264df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## Current ### Features +- [#2667](https://github.com/poanetwork/blockscout/pull/2667) - Add ETS-based cache for accounts page - [#2679](https://github.com/poanetwork/blockscout/pull/2679) - added fixed height for card chain blocks and card chain transactions - [#2678](https://github.com/poanetwork/blockscout/pull/2678) - fixed dashboard banner height bug - [#2672](https://github.com/poanetwork/blockscout/pull/2672) - added new theme for xUSDT diff --git a/apps/block_scout_web/test/support/conn_case.ex b/apps/block_scout_web/test/support/conn_case.ex index 4ab8f3bcd9..ed3f094c34 100644 --- a/apps/block_scout_web/test/support/conn_case.ex +++ b/apps/block_scout_web/test/support/conn_case.ex @@ -42,6 +42,8 @@ defmodule BlockScoutWeb.ConnCase do Supervisor.terminate_child(Explorer.Supervisor, Explorer.Chain.Cache.Transactions.child_id()) Supervisor.restart_child(Explorer.Supervisor, Explorer.Chain.Cache.Transactions.child_id()) + Supervisor.terminate_child(Explorer.Supervisor, Explorer.Chain.Cache.Accounts.child_id()) + Supervisor.restart_child(Explorer.Supervisor, Explorer.Chain.Cache.Accounts.child_id()) {:ok, conn: Phoenix.ConnTest.build_conn()} end diff --git a/apps/block_scout_web/test/support/feature_case.ex b/apps/block_scout_web/test/support/feature_case.ex index cdacd547bf..f6b1aedf24 100644 --- a/apps/block_scout_web/test/support/feature_case.ex +++ b/apps/block_scout_web/test/support/feature_case.ex @@ -29,6 +29,8 @@ defmodule BlockScoutWeb.FeatureCase do Supervisor.terminate_child(Explorer.Supervisor, Explorer.Chain.Cache.Transactions.child_id()) Supervisor.restart_child(Explorer.Supervisor, Explorer.Chain.Cache.Transactions.child_id()) + Supervisor.terminate_child(Explorer.Supervisor, Explorer.Chain.Cache.Accounts.child_id()) + Supervisor.restart_child(Explorer.Supervisor, Explorer.Chain.Cache.Accounts.child_id()) metadata = Phoenix.Ecto.SQL.Sandbox.metadata_for(Explorer.Repo, self()) {:ok, session} = Wallaby.start_session(metadata: metadata) diff --git a/apps/explorer/config/config.exs b/apps/explorer/config/config.exs index 8a264eaf33..48d9b55610 100644 --- a/apps/explorer/config/config.exs +++ b/apps/explorer/config/config.exs @@ -136,6 +136,10 @@ config :explorer, Explorer.Chain.Cache.Transactions, ttl_check_interval: if(System.get_env("DISABLE_INDEXER") == "true", do: :timer.seconds(1), else: false), global_ttl: if(System.get_env("DISABLE_INDEXER") == "true", do: :timer.seconds(5)) +config :explorer, Explorer.Chain.Cache.Accounts, + ttl_check_interval: if(System.get_env("DISABLE_INDEXER") == "true", do: :timer.seconds(1), else: false), + global_ttl: if(System.get_env("DISABLE_INDEXER") == "true", do: :timer.seconds(5)) + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" diff --git a/apps/explorer/lib/explorer/application.ex b/apps/explorer/lib/explorer/application.ex index 7c55d28f76..9645572b62 100644 --- a/apps/explorer/lib/explorer/application.ex +++ b/apps/explorer/lib/explorer/application.ex @@ -8,6 +8,7 @@ defmodule Explorer.Application do alias Explorer.Admin alias Explorer.Chain.Cache.{ + Accounts, BlockCount, BlockNumber, Blocks, @@ -49,7 +50,8 @@ defmodule Explorer.Application do BlockNumber, con_cache_child_spec(MarketHistoryCache.cache_name()), con_cache_child_spec(RSK.cache_name(), ttl_check_interval: :timer.minutes(1), global_ttl: :timer.minutes(30)), - Transactions + Transactions, + Accounts ] children = base_children ++ configurable_children() diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index 6de1adc9ff..d39945034d 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -49,6 +49,7 @@ defmodule Explorer.Chain do alias Explorer.Chain.Block.{EmissionReward, Reward} alias Explorer.Chain.Cache.{ + Accounts, BlockCount, BlockNumber, Blocks, @@ -1379,6 +1380,36 @@ defmodule Explorer.Chain do def list_top_addresses(options \\ []) do paging_options = Keyword.get(options, :paging_options, @default_paging_options) + if is_nil(paging_options.key) do + paging_options.page_size + |> Accounts.take_enough() + |> case do + nil -> + accounts_with_n = fetch_top_addresses(paging_options) + + accounts_with_n + |> Enum.map(fn {address, _n} -> address end) + |> Accounts.update() + + accounts_with_n + + accounts -> + Enum.map( + accounts, + &{&1, + if is_nil(&1.nonce) do + 0 + else + &1.nonce + 1 + end} + ) + end + else + fetch_top_addresses(paging_options) + end + end + + defp fetch_top_addresses(paging_options) do base_query = from(a in Address, where: a.fetched_coin_balance > ^0, diff --git a/apps/explorer/lib/explorer/chain/cache/accounts.ex b/apps/explorer/lib/explorer/chain/cache/accounts.ex new file mode 100644 index 0000000000..523bd44877 --- /dev/null +++ b/apps/explorer/lib/explorer/chain/cache/accounts.ex @@ -0,0 +1,73 @@ +defmodule Explorer.Chain.Cache.Accounts do + @moduledoc """ + Caches the top Addresses + """ + + alias Explorer.Chain.Address + + use Explorer.Chain.OrderedCache, + name: :accounts, + max_size: 51, + preload: :names, + ttl_check_interval: Application.get_env(:explorer, __MODULE__)[:ttl_check_interval], + global_ttl: Application.get_env(:explorer, __MODULE__)[:global_ttl] + + @type element :: Address.t() + + @type id :: {non_neg_integer(), non_neg_integer()} + + def element_to_id(%Address{fetched_coin_balance: fetched_coin_balance, hash: hash}) do + {fetched_coin_balance, hash} + end + + def prevails?({fetched_coin_balance_a, hash_a}, {fetched_coin_balance_b, hash_b}) do + # same as a query's `order_by: [desc: :fetched_coin_balance, asc: :hash]` + if fetched_coin_balance_a == fetched_coin_balance_b do + hash_a < hash_b + else + fetched_coin_balance_a > fetched_coin_balance_b + end + end + + def drop(nil), do: :ok + + def drop([]), do: :ok + + def drop(addresses) when is_list(addresses) do + # This has to be used by the Indexer insead of `update`. + # The reason being that addresses already in the cache can change their balance + # value and removing or updating them will result into a potentially invalid + # cache status, that would not even get corrected with time. + # The only thing we can safely do when an address in the cache changes its + # `fetched_coin_balance` is to invalidate the whole cache and wait for it + # to be filled again (by the query that it takes the place of when full). + ConCache.update(cache_name(), ids_list_key(), fn ids -> + if drop_needed?(ids, addresses) do + # Remove the addresses immediately + Enum.each(ids, &ConCache.delete(cache_name(), &1)) + + {:ok, []} + else + {:ok, ids} + end + end) + end + + def drop(address), do: drop([address]) + + defp drop_needed?(ids, _addresses) when is_nil(ids), do: false + + defp drop_needed?([], _addresses), do: false + + defp drop_needed?(ids, addresses) do + ids_map = Map.new(ids, fn {balance, hash} -> {hash, balance} end) + + # Result it `true` only when the address is present in the cache already, + # but with a different `fetched_coin_balance` + Enum.find_value(addresses, false, fn address -> + stored_address_balance = Map.get(ids_map, address.hash) + + stored_address_balance && stored_address_balance != address.fetched_coin_balance + end) + end +end diff --git a/apps/explorer/test/explorer/chain/cache/accounts_test.exs b/apps/explorer/test/explorer/chain/cache/accounts_test.exs new file mode 100644 index 0000000000..a1a49bd67b --- /dev/null +++ b/apps/explorer/test/explorer/chain/cache/accounts_test.exs @@ -0,0 +1,42 @@ +defmodule Explorer.Chain.Cache.AccountsTest do + use Explorer.DataCase + + alias Explorer.Chain.Cache.Accounts + alias Explorer.Repo + + describe "drop/1" do + test "does not drop the cache if the address fetched_coin_balance has not changed" do + address = + insert(:address, fetched_coin_balance: 100_000, fetched_coin_balance_block_number: 1) + |> preload_names() + + Accounts.update(address) + + assert Accounts.take(1) == [address] + + Accounts.drop(address) + + assert Accounts.take(1) == [address] + end + + test "drops the cache if an address was in the cache with a different fetched_coin_balance" do + address = + insert(:address, fetched_coin_balance: 100_000, fetched_coin_balance_block_number: 1) + |> preload_names() + + Accounts.update(address) + + assert Accounts.take(1) == [address] + + updated_address = %{address | fetched_coin_balance: 100_001} + + Accounts.drop(updated_address) + + assert Accounts.take(1) == [] + end + end + + defp preload_names(address) do + Repo.preload(address, [:names]) + end +end diff --git a/apps/explorer/test/support/data_case.ex b/apps/explorer/test/support/data_case.ex index 47c15bbca0..e12dd24a82 100644 --- a/apps/explorer/test/support/data_case.ex +++ b/apps/explorer/test/support/data_case.ex @@ -45,6 +45,8 @@ defmodule Explorer.DataCase do Supervisor.restart_child(Explorer.Supervisor, Explorer.Chain.Cache.Blocks.child_id()) Supervisor.terminate_child(Explorer.Supervisor, Explorer.Chain.Cache.Transactions.child_id()) Supervisor.restart_child(Explorer.Supervisor, Explorer.Chain.Cache.Transactions.child_id()) + Supervisor.terminate_child(Explorer.Supervisor, Explorer.Chain.Cache.Accounts.child_id()) + Supervisor.restart_child(Explorer.Supervisor, Explorer.Chain.Cache.Accounts.child_id()) :ok end diff --git a/apps/indexer/lib/indexer/block/fetcher.ex b/apps/indexer/lib/indexer/block/fetcher.ex index 28002f010a..1b95d84086 100644 --- a/apps/indexer/lib/indexer/block/fetcher.ex +++ b/apps/indexer/lib/indexer/block/fetcher.ex @@ -13,7 +13,7 @@ defmodule Indexer.Block.Fetcher do alias Explorer.Chain alias Explorer.Chain.{Address, Block, Hash, Import, Transaction} alias Explorer.Chain.Cache.Blocks, as: BlocksCache - alias Explorer.Chain.Cache.{BlockNumber, Transactions} + alias Explorer.Chain.Cache.{Accounts, BlockNumber, Transactions} alias Indexer.Block.Fetcher.Receipts alias Indexer.Fetcher.{ @@ -176,6 +176,7 @@ defmodule Indexer.Block.Fetcher do result = {:ok, %{inserted: inserted, errors: blocks_errors}} update_block_cache(inserted[:blocks]) update_transactions_cache(inserted[:transactions]) + update_addresses_cache(inserted[:addresses]) result else {step, {:error, reason}} -> {:error, {step, reason}} @@ -197,6 +198,8 @@ defmodule Indexer.Block.Fetcher do Transactions.update(transactions) end + defp update_addresses_cache(addresses), do: Accounts.drop(addresses) + def import( %__MODULE__{broadcast: broadcast, callback_module: callback_module} = state, options diff --git a/apps/indexer/lib/indexer/block/realtime/fetcher.ex b/apps/indexer/lib/indexer/block/realtime/fetcher.ex index 8edd87738d..897eab2cff 100644 --- a/apps/indexer/lib/indexer/block/realtime/fetcher.ex +++ b/apps/indexer/lib/indexer/block/realtime/fetcher.ex @@ -27,6 +27,7 @@ defmodule Indexer.Block.Realtime.Fetcher do alias Ecto.Changeset alias EthereumJSONRPC.{FetchedBalances, Subscription} alias Explorer.Chain + alias Explorer.Chain.Cache.Accounts alias Explorer.Counters.AverageBlockTime alias Indexer.{Block, Tracer} alias Indexer.Block.Realtime.TaskSupervisor @@ -197,6 +198,8 @@ defmodule Indexer.Block.Realtime.Fetcher do json_rpc_named_arguments ) + Accounts.drop(imported[:addresses]) + ok end end diff --git a/apps/indexer/lib/indexer/fetcher/block_reward.ex b/apps/indexer/lib/indexer/fetcher/block_reward.ex index 4a59eb1cbe..ed593bb655 100644 --- a/apps/indexer/lib/indexer/fetcher/block_reward.ex +++ b/apps/indexer/lib/indexer/fetcher/block_reward.ex @@ -17,6 +17,7 @@ defmodule Indexer.Fetcher.BlockReward do alias EthereumJSONRPC.FetchedBeneficiaries alias Explorer.Chain alias Explorer.Chain.{Block, Wei} + alias Explorer.Chain.Cache.Accounts alias Indexer.{BufferedTask, Tracer} alias Indexer.Fetcher.BlockReward.Supervisor, as: BlockRewardSupervisor alias Indexer.Fetcher.CoinBalance @@ -130,7 +131,9 @@ defmodule Indexer.Fetcher.BlockReward do |> add_gas_payments() |> import_block_reward_params() |> case do - {:ok, %{address_coin_balances: address_coin_balances}} -> + {:ok, %{address_coin_balances: address_coin_balances, addresses: addresses}} -> + Accounts.drop(addresses) + CoinBalance.async_fetch_balances(address_coin_balances) retry_errors(errors) diff --git a/apps/indexer/lib/indexer/fetcher/coin_balance.ex b/apps/indexer/lib/indexer/fetcher/coin_balance.ex index e9930d0e0e..f8fadfcb53 100644 --- a/apps/indexer/lib/indexer/fetcher/coin_balance.ex +++ b/apps/indexer/lib/indexer/fetcher/coin_balance.ex @@ -14,6 +14,7 @@ defmodule Indexer.Fetcher.CoinBalance do alias EthereumJSONRPC.FetchedBalances alias Explorer.Chain alias Explorer.Chain.{Block, Hash} + alias Explorer.Chain.Cache.Accounts alias Indexer.{BufferedTask, Tracer} @behaviour BufferedTask @@ -136,7 +137,9 @@ defmodule Indexer.Fetcher.CoinBalance do end defp run_fetched_balances(%FetchedBalances{errors: errors} = fetched_balances, _) do - {:ok, _} = import_fetched_balances(fetched_balances) + {:ok, imported} = import_fetched_balances(fetched_balances) + + Accounts.drop(imported[:addresses]) retry(errors) end diff --git a/apps/indexer/lib/indexer/fetcher/coin_balance_on_demand.ex b/apps/indexer/lib/indexer/fetcher/coin_balance_on_demand.ex index d6d7d66d71..17014d2ede 100644 --- a/apps/indexer/lib/indexer/fetcher/coin_balance_on_demand.ex +++ b/apps/indexer/lib/indexer/fetcher/coin_balance_on_demand.ex @@ -19,7 +19,7 @@ defmodule Indexer.Fetcher.CoinBalanceOnDemand do alias Explorer.{Chain, Repo} alias Explorer.Chain.Address alias Explorer.Chain.Address.CoinBalance - alias Explorer.Chain.Cache.BlockNumber + alias Explorer.Chain.Cache.{Accounts, BlockNumber} alias Explorer.Counters.AverageBlockTime alias Indexer.Fetcher.CoinBalance, as: CoinBalanceFetcher alias Timex.Duration @@ -71,7 +71,11 @@ defmodule Indexer.Fetcher.CoinBalanceOnDemand do end def handle_cast({:fetch_and_update, block_number, address}, state) do - fetch_and_update(block_number, address, state.json_rpc_named_arguments) + result = fetch_and_update(block_number, address, state.json_rpc_named_arguments) + + with {:ok, %{addresses: addresses}} <- result do + Accounts.drop(addresses) + end {:noreply, state} end diff --git a/apps/indexer/lib/indexer/fetcher/contract_code.ex b/apps/indexer/lib/indexer/fetcher/contract_code.ex index bc95dd0d8e..cd9a5baa08 100644 --- a/apps/indexer/lib/indexer/fetcher/contract_code.ex +++ b/apps/indexer/lib/indexer/fetcher/contract_code.ex @@ -12,6 +12,7 @@ defmodule Indexer.Fetcher.ContractCode do alias Explorer.Chain alias Explorer.Chain.{Block, Hash} + alias Explorer.Chain.Cache.Accounts alias Indexer.{BufferedTask, Tracer} alias Indexer.Transform.Addresses @@ -126,7 +127,8 @@ defmodule Indexer.Fetcher.ContractCode do addresses: %{params: merged_addresses_params}, timeout: :infinity }) do - {:ok, _} -> + {:ok, imported} -> + Accounts.drop(imported[:addresses]) :ok {:error, step, reason, _changes_so_far} -> diff --git a/apps/indexer/lib/indexer/fetcher/internal_transaction.ex b/apps/indexer/lib/indexer/fetcher/internal_transaction.ex index 7d93bbf64f..5ad7c5d506 100644 --- a/apps/indexer/lib/indexer/fetcher/internal_transaction.ex +++ b/apps/indexer/lib/indexer/fetcher/internal_transaction.ex @@ -14,6 +14,7 @@ defmodule Indexer.Fetcher.InternalTransaction do alias Explorer.Chain alias Explorer.Chain.{Block, Hash} + alias Explorer.Chain.Cache.Accounts alias Indexer.{BufferedTask, Tracer} alias Indexer.Transform.Addresses @@ -218,6 +219,8 @@ defmodule Indexer.Fetcher.InternalTransaction do case imports do {:ok, imported} -> + Accounts.drop(imported[:addreses]) + async_import_coin_balances(imported, %{ address_hash_to_fetched_balance_block_number: address_hash_to_block_number }) diff --git a/apps/indexer/lib/indexer/fetcher/pending_transaction.ex b/apps/indexer/lib/indexer/fetcher/pending_transaction.ex index 7576d384d8..176d8dc9b7 100644 --- a/apps/indexer/lib/indexer/fetcher/pending_transaction.ex +++ b/apps/indexer/lib/indexer/fetcher/pending_transaction.ex @@ -14,6 +14,7 @@ defmodule Indexer.Fetcher.PendingTransaction do alias Ecto.Changeset alias Explorer.Chain + alias Explorer.Chain.Cache.Accounts alias Indexer.Fetcher.PendingTransaction alias Indexer.Transform.Addresses @@ -148,7 +149,8 @@ defmodule Indexer.Fetcher.PendingTransaction do broadcast: :realtime, transactions: %{params: transactions_params, on_conflict: :nothing} }) do - {:ok, _} -> + {:ok, imported} -> + Accounts.drop(imported[:addresses]) :ok {:error, [%Changeset{} | _] = changesets} -> diff --git a/apps/indexer/lib/indexer/fetcher/uncle_block.ex b/apps/indexer/lib/indexer/fetcher/uncle_block.ex index e9300d1e39..e597d16ca2 100644 --- a/apps/indexer/lib/indexer/fetcher/uncle_block.ex +++ b/apps/indexer/lib/indexer/fetcher/uncle_block.ex @@ -12,6 +12,7 @@ defmodule Indexer.Fetcher.UncleBlock do alias Ecto.Changeset alias EthereumJSONRPC.Blocks alias Explorer.Chain + alias Explorer.Chain.Cache.Accounts alias Explorer.Chain.Hash alias Indexer.{Block, BufferedTask, Tracer} alias Indexer.Fetcher.UncleBlock @@ -126,7 +127,8 @@ defmodule Indexer.Fetcher.UncleBlock do block_second_degree_relations: %{params: block_second_degree_relations_params}, transactions: %{params: transactions_params, on_conflict: :nothing} }) do - {:ok, _} -> + {:ok, imported} -> + Accounts.drop(imported[:addresses]) retry(errors) {:error, {:import = step, [%Changeset{} | _] = changesets}} -> From 432e6419dfcac6d7a1a400608cfa8f671824f708 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Wed, 18 Sep 2019 17:36:27 +0300 Subject: [PATCH 08/15] add comment --- apps/explorer/lib/explorer/chain.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index 1951f500a2..ce6995fc01 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -3074,6 +3074,7 @@ defmodule Explorer.Chain do |> normalize_balances_by_day() end + # https://github.com/poanetwork/blockscout/issues/2658 defp replace_last_value(items, %{value: value, timestamp: timestamp}) do List.replace_at(items, -1, %{date: Date.convert!(timestamp, Calendar.ISO), value: value}) end From 24d606dbed40203de7b4ef58cea6afb5e164d939 Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Wed, 18 Sep 2019 17:40:34 +0300 Subject: [PATCH 09/15] Fix stuck label and value for uncle block height --- CHANGELOG.md | 1 + .../templates/block/overview.html.eex | 3 +- apps/block_scout_web/priv/gettext/default.pot | 28 +++++++++---------- .../priv/gettext/en/LC_MESSAGES/default.po | 28 +++++++++---------- 4 files changed, 30 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 30f508dbd2..d98f2c0141 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ - [#2663](https://github.com/poanetwork/blockscout/pull/2663) - Fetch address counters in parallel ### Fixes +- [#2709](https://github.com/poanetwork/blockscout/pull/2709) - Fix stuck label and value for uncle block height - [#2707](https://github.com/poanetwork/blockscout/pull/2707) - fix for dashboard banner chart legend items - [#2701](https://github.com/poanetwork/blockscout/pull/2701) - Exclude nonconsensus blocks from avg block time calculation by default - [#2696](https://github.com/poanetwork/blockscout/pull/2696) - do not update fetched_coin_balance with nil diff --git a/apps/block_scout_web/lib/block_scout_web/templates/block/overview.html.eex b/apps/block_scout_web/lib/block_scout_web/templates/block/overview.html.eex index 752fd314e3..c2774ce4bd 100644 --- a/apps/block_scout_web/lib/block_scout_web/templates/block/overview.html.eex +++ b/apps/block_scout_web/lib/block_scout_web/templates/block/overview.html.eex @@ -12,8 +12,7 @@ <%= if block_type(@block) == "Block" do %> <%= gettext("Block Height: %{height}", height: @block.number) %> <%= if @block.number == 0, do: "- " <> gettext("Genesis Block")%> <% else %> - <%= gettext("%{block_type} Height:", block_type: block_type(@block)) %> - <%= link(@block, to: block_path(BlockScoutWeb.Endpoint, :show, @block.number)) %> + <%= gettext("%{block_type} Height:", block_type: block_type(@block)) %> <%= link(@block, to: block_path(BlockScoutWeb.Endpoint, :show, @block.number)) %> <% end %>
diff --git a/apps/block_scout_web/priv/gettext/default.pot b/apps/block_scout_web/priv/gettext/default.pot index 8a1582b2e7..dfe785c39b 100644 --- a/apps/block_scout_web/priv/gettext/default.pot +++ b/apps/block_scout_web/priv/gettext/default.pot @@ -43,7 +43,7 @@ msgid "%{block_type}s" msgstr "" #, elixir-format -#: lib/block_scout_web/templates/block/overview.html.eex:21 +#: lib/block_scout_web/templates/block/overview.html.eex:20 #: lib/block_scout_web/templates/chain/_block.html.eex:11 msgid "%{count} Transactions" msgstr "" @@ -480,7 +480,7 @@ msgid "Copy Txn Hash" msgstr "" #, elixir-format -#: lib/block_scout_web/templates/block/overview.html.eex:59 +#: lib/block_scout_web/templates/block/overview.html.eex:58 msgid "Difficulty" msgstr "" @@ -562,15 +562,15 @@ msgstr "" #, elixir-format #: lib/block_scout_web/templates/block/_tile.html.eex:56 -#: lib/block_scout_web/templates/block/overview.html.eex:109 -#: lib/block_scout_web/templates/block/overview.html.eex:159 +#: lib/block_scout_web/templates/block/overview.html.eex:108 +#: lib/block_scout_web/templates/block/overview.html.eex:158 msgid "Gas Limit" msgstr "" #, elixir-format #: lib/block_scout_web/templates/block/_tile.html.eex:61 -#: lib/block_scout_web/templates/block/overview.html.eex:102 -#: lib/block_scout_web/templates/block/overview.html.eex:153 +#: lib/block_scout_web/templates/block/overview.html.eex:101 +#: lib/block_scout_web/templates/block/overview.html.eex:152 msgid "Gas Used" msgstr "" @@ -657,7 +657,7 @@ msgstr "" #, elixir-format #: lib/block_scout_web/templates/block/_tile.html.eex:38 -#: lib/block_scout_web/templates/block/overview.html.eex:125 +#: lib/block_scout_web/templates/block/overview.html.eex:124 #: lib/block_scout_web/templates/chain/_block.html.eex:15 msgid "Miner" msgstr "" @@ -712,7 +712,7 @@ msgid "Execute" msgstr "" #, elixir-format -#: lib/block_scout_web/templates/block/overview.html.eex:74 +#: lib/block_scout_web/templates/block/overview.html.eex:73 #: lib/block_scout_web/templates/transaction/overview.html.eex:79 msgid "Nonce" msgstr "" @@ -767,7 +767,7 @@ msgid "GET" msgstr "" #, elixir-format -#: lib/block_scout_web/templates/block/overview.html.eex:85 +#: lib/block_scout_web/templates/block/overview.html.eex:84 msgid "Position %{index}" msgstr "" @@ -798,7 +798,7 @@ msgid "Gwei" msgstr "" #, elixir-format -#: lib/block_scout_web/templates/block/overview.html.eex:37 +#: lib/block_scout_web/templates/block/overview.html.eex:36 msgid "Hash" msgstr "" @@ -964,7 +964,7 @@ msgid "Top Accounts - %{subnetwork} Explorer" msgstr "" #, elixir-format -#: lib/block_scout_web/templates/block/overview.html.eex:68 +#: lib/block_scout_web/templates/block/overview.html.eex:67 msgid "Total Difficulty" msgstr "" @@ -1246,7 +1246,7 @@ msgid "Parameters" msgstr "" #, elixir-format -#: lib/block_scout_web/templates/block/overview.html.eex:44 +#: lib/block_scout_web/templates/block/overview.html.eex:43 msgid "Parent Hash" msgstr "" @@ -1404,7 +1404,7 @@ msgid "Show Validator Info" msgstr "" #, elixir-format -#: lib/block_scout_web/templates/block/overview.html.eex:146 +#: lib/block_scout_web/templates/block/overview.html.eex:145 msgid "Block Rewards" msgstr "" @@ -1646,7 +1646,7 @@ msgid "Uncle Reward" msgstr "" #, elixir-format -#: lib/block_scout_web/templates/block/overview.html.eex:81 +#: lib/block_scout_web/templates/block/overview.html.eex:80 #: lib/block_scout_web/templates/layout/_topnav.html.eex:52 msgid "Uncles" msgstr "" diff --git a/apps/block_scout_web/priv/gettext/en/LC_MESSAGES/default.po b/apps/block_scout_web/priv/gettext/en/LC_MESSAGES/default.po index 8a1582b2e7..dfe785c39b 100644 --- a/apps/block_scout_web/priv/gettext/en/LC_MESSAGES/default.po +++ b/apps/block_scout_web/priv/gettext/en/LC_MESSAGES/default.po @@ -43,7 +43,7 @@ msgid "%{block_type}s" msgstr "" #, elixir-format -#: lib/block_scout_web/templates/block/overview.html.eex:21 +#: lib/block_scout_web/templates/block/overview.html.eex:20 #: lib/block_scout_web/templates/chain/_block.html.eex:11 msgid "%{count} Transactions" msgstr "" @@ -480,7 +480,7 @@ msgid "Copy Txn Hash" msgstr "" #, elixir-format -#: lib/block_scout_web/templates/block/overview.html.eex:59 +#: lib/block_scout_web/templates/block/overview.html.eex:58 msgid "Difficulty" msgstr "" @@ -562,15 +562,15 @@ msgstr "" #, elixir-format #: lib/block_scout_web/templates/block/_tile.html.eex:56 -#: lib/block_scout_web/templates/block/overview.html.eex:109 -#: lib/block_scout_web/templates/block/overview.html.eex:159 +#: lib/block_scout_web/templates/block/overview.html.eex:108 +#: lib/block_scout_web/templates/block/overview.html.eex:158 msgid "Gas Limit" msgstr "" #, elixir-format #: lib/block_scout_web/templates/block/_tile.html.eex:61 -#: lib/block_scout_web/templates/block/overview.html.eex:102 -#: lib/block_scout_web/templates/block/overview.html.eex:153 +#: lib/block_scout_web/templates/block/overview.html.eex:101 +#: lib/block_scout_web/templates/block/overview.html.eex:152 msgid "Gas Used" msgstr "" @@ -657,7 +657,7 @@ msgstr "" #, elixir-format #: lib/block_scout_web/templates/block/_tile.html.eex:38 -#: lib/block_scout_web/templates/block/overview.html.eex:125 +#: lib/block_scout_web/templates/block/overview.html.eex:124 #: lib/block_scout_web/templates/chain/_block.html.eex:15 msgid "Miner" msgstr "" @@ -712,7 +712,7 @@ msgid "Execute" msgstr "" #, elixir-format -#: lib/block_scout_web/templates/block/overview.html.eex:74 +#: lib/block_scout_web/templates/block/overview.html.eex:73 #: lib/block_scout_web/templates/transaction/overview.html.eex:79 msgid "Nonce" msgstr "" @@ -767,7 +767,7 @@ msgid "GET" msgstr "" #, elixir-format -#: lib/block_scout_web/templates/block/overview.html.eex:85 +#: lib/block_scout_web/templates/block/overview.html.eex:84 msgid "Position %{index}" msgstr "" @@ -798,7 +798,7 @@ msgid "Gwei" msgstr "" #, elixir-format -#: lib/block_scout_web/templates/block/overview.html.eex:37 +#: lib/block_scout_web/templates/block/overview.html.eex:36 msgid "Hash" msgstr "" @@ -964,7 +964,7 @@ msgid "Top Accounts - %{subnetwork} Explorer" msgstr "" #, elixir-format -#: lib/block_scout_web/templates/block/overview.html.eex:68 +#: lib/block_scout_web/templates/block/overview.html.eex:67 msgid "Total Difficulty" msgstr "" @@ -1246,7 +1246,7 @@ msgid "Parameters" msgstr "" #, elixir-format -#: lib/block_scout_web/templates/block/overview.html.eex:44 +#: lib/block_scout_web/templates/block/overview.html.eex:43 msgid "Parent Hash" msgstr "" @@ -1404,7 +1404,7 @@ msgid "Show Validator Info" msgstr "" #, elixir-format -#: lib/block_scout_web/templates/block/overview.html.eex:146 +#: lib/block_scout_web/templates/block/overview.html.eex:145 msgid "Block Rewards" msgstr "" @@ -1646,7 +1646,7 @@ msgid "Uncle Reward" msgstr "" #, elixir-format -#: lib/block_scout_web/templates/block/overview.html.eex:81 +#: lib/block_scout_web/templates/block/overview.html.eex:80 #: lib/block_scout_web/templates/layout/_topnav.html.eex:52 msgid "Uncles" msgstr "" From 0944ea5cdcc186f35ed5d89eebf151e7da9f6d32 Mon Sep 17 00:00:00 2001 From: pasqu4le Date: Fri, 20 Sep 2019 18:01:19 +0200 Subject: [PATCH 10/15] Improve speed of nonconsensus data removal Problem: removal of nonconsensus data is too inefficient and as a result blocks are imported too slow. Solution: reformulation of deletion logic for better performance --- .../explorer/chain/import/runner/blocks.ex | 246 ++++++------------ .../chain/import/runner/blocks_test.exs | 12 +- 2 files changed, 91 insertions(+), 167 deletions(-) diff --git a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex index f893df7684..394e21cbaa 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex @@ -46,32 +46,14 @@ defmodule Explorer.Chain.Import.Runner.Blocks do hashes = Enum.map(changes_list, & &1.hash) consensus_block_numbers = consensus_block_numbers(changes_list) - where_invalid_neighbour = where_invalid_neighbour(changes_list) # Enforce ShareLocks tables order (see docs: sharelocks.md) multi - |> Multi.run(:acquire_blocks, fn repo, _ -> - acquire_blocks(repo, hashes, consensus_block_numbers, where_invalid_neighbour) - end) |> Multi.run(:lose_consensus, fn repo, _ -> - lose_consensus(repo, consensus_block_numbers, insert_options) - end) - |> Multi.run(:lose_invalid_neighbour_consensus, fn repo, _ -> - lose_invalid_neighbour_consensus(repo, where_invalid_neighbour, insert_options) - end) - |> Multi.run(:nonconsensus_block_numbers, fn _repo, - %{ - lose_consensus: lost_consensus_blocks, - lose_invalid_neighbour_consensus: lost_consensus_neighbours - } -> - nonconsensus_block_numbers = - (lost_consensus_blocks ++ lost_consensus_neighbours) - |> Enum.sort() - |> Enum.dedup() - - {:ok, nonconsensus_block_numbers} + lose_consensus(repo, hashes, consensus_block_numbers, changes_list, insert_options) end) |> Multi.run(:blocks, fn repo, _ -> + # Note, needs to be executed after `lose_consensus` for lock acquisition insert(repo, changes_list, insert_options) end) |> Multi.run(:uncle_fetched_block_second_degree_relations, fn repo, %{blocks: blocks} when is_list(blocks) -> @@ -101,27 +83,14 @@ defmodule Explorer.Chain.Import.Runner.Blocks do transactions: transactions }) end) - |> Multi.run(:remove_nonconsensus_logs, fn repo, - %{ - nonconsensus_block_numbers: nonconsensus_block_numbers, - fork_transactions: transactions - } -> - remove_nonconsensus_logs(repo, nonconsensus_block_numbers, transactions, insert_options) + |> Multi.run(:remove_nonconsensus_logs, fn repo, %{derive_transaction_forks: transactions} -> + remove_nonconsensus_logs(repo, transactions, insert_options) end) - |> Multi.run(:acquire_internal_transactions, fn repo, - %{ - nonconsensus_block_numbers: nonconsensus_block_numbers, - fork_transactions: transactions - } -> - acquire_internal_transactions(repo, nonconsensus_block_numbers, hashes, transactions) + |> Multi.run(:acquire_internal_transactions, fn repo, %{derive_transaction_forks: transactions} -> + acquire_internal_transactions(repo, hashes, transactions) end) - |> Multi.run(:remove_nonconsensus_internal_transactions, fn repo, - %{ - nonconsensus_block_numbers: - nonconsensus_block_numbers, - fork_transactions: transactions - } -> - remove_nonconsensus_internal_transactions(repo, nonconsensus_block_numbers, transactions, insert_options) + |> Multi.run(:remove_nonconsensus_internal_transactions, fn repo, %{derive_transaction_forks: transactions} -> + remove_nonconsensus_internal_transactions(repo, transactions, insert_options) end) |> Multi.run(:internal_transaction_transaction_block_number, fn repo, _ -> update_internal_transaction_block_number(repo, hashes) @@ -129,9 +98,8 @@ defmodule Explorer.Chain.Import.Runner.Blocks do |> Multi.run(:acquire_contract_address_tokens, fn repo, _ -> acquire_contract_address_tokens(repo, consensus_block_numbers) end) - |> Multi.run(:remove_nonconsensus_token_transfers, fn repo, - %{nonconsensus_block_numbers: nonconsensus_block_numbers} -> - remove_nonconsensus_token_transfers(repo, nonconsensus_block_numbers, insert_options) + |> Multi.run(:remove_nonconsensus_token_transfers, fn repo, %{derive_transaction_forks: transactions} -> + remove_nonconsensus_token_transfers(repo, transactions, insert_options) end) |> Multi.run(:delete_address_token_balances, fn repo, _ -> delete_address_token_balances(repo, consensus_block_numbers, insert_options) @@ -159,22 +127,6 @@ defmodule Explorer.Chain.Import.Runner.Blocks do @impl Runner def timeout, do: @timeout - defp acquire_blocks(repo, hashes, consensus_block_numbers, where_invalid_neighbour) do - query = - from( - block in where_invalid_neighbour, - or_where: block.number in ^consensus_block_numbers, - or_where: block.hash in ^hashes, - select: block.hash, - # Enforce Block ShareLocks order (see docs: sharelocks.md) - order_by: [asc: block.hash], - lock: "FOR UPDATE" - ) - - blocks = repo.all(query) - {:ok, blocks} - end - defp acquire_contract_address_tokens(repo, consensus_block_numbers) do query = from(address_current_token_balance in Address.CurrentTokenBalance, @@ -187,15 +139,12 @@ defmodule Explorer.Chain.Import.Runner.Blocks do Tokens.acquire_contract_address_tokens(repo, contract_address_hashes) end - defp acquire_internal_transactions(repo, nonconsensus_block_numbers, hashes, forked_transactions) do - forked_transaction_hashes = Enum.map(forked_transactions, & &1.hash) - + defp acquire_internal_transactions(repo, hashes, forked_transaction_hashes) do query = from(internal_transaction in InternalTransaction, join: transaction in Transaction, on: internal_transaction.transaction_hash == transaction.hash, - where: transaction.block_number in ^nonconsensus_block_numbers, - or_where: transaction.block_hash in ^hashes, + where: transaction.block_hash in ^hashes, or_where: transaction.hash in ^forked_transaction_hashes, select: {internal_transaction.transaction_hash, internal_transaction.index}, # Enforce InternalTransaction ShareLocks order (see docs: sharelocks.md) @@ -229,14 +178,11 @@ defmodule Explorer.Chain.Import.Runner.Blocks do lock: "FOR UPDATE" ) - transactions = repo.all(query) - - hashes = Enum.map(transactions, & &1.hash) - update_query = from( t in Transaction, - where: t.hash in ^hashes, + join: s in subquery(query), + on: t.hash == s.hash, update: [ set: [ block_hash: nil, @@ -250,17 +196,19 @@ defmodule Explorer.Chain.Import.Runner.Blocks do updated_at: ^updated_at ] ], - select: t.hash + select: %{ + block_hash: s.block_hash, + index: s.index, + hash: s.hash + } ) - try do - {_num, _res} = repo.update_all(update_query, [], timeout: timeout) + {_num, transactions} = repo.update_all(update_query, [], timeout: timeout) - {:ok, transactions} - rescue - postgrex_error in Postgrex.Error -> - {:error, %{exception: postgrex_error}} - end + {:ok, transactions} + rescue + postgrex_error in Postgrex.Error -> + {:error, %{exception: postgrex_error}} end defp derive_transaction_forks(%{ @@ -283,7 +231,7 @@ defmodule Explorer.Chain.Import.Runner.Blocks do # Enforce Fork ShareLocks order (see docs: sharelocks.md) |> Enum.sort_by(&{&1.uncle_hash, &1.index}) - {_total, result} = + {_total, forked_transaction} = repo.insert_all( Transaction.Fork, transaction_forks, @@ -294,11 +242,11 @@ defmodule Explorer.Chain.Import.Runner.Blocks do update: [set: [hash: fragment("EXCLUDED.hash")]], where: fragment("EXCLUDED.hash <> ?", transaction_fork.hash) ), - returning: [:uncle_hash, :hash], + returning: [:hash], timeout: timeout ) - {:ok, result} + {:ok, Enum.map(forked_transaction, & &1.hash)} end @spec insert(Repo.t(), [map()], %{ @@ -364,47 +312,48 @@ defmodule Explorer.Chain.Import.Runner.Blocks do |> Enum.map(& &1.number) end - defp lose_consensus(_, [], _), do: {:ok, []} - - defp lose_consensus(repo, consensus_block_number, %{timeout: timeout, timestamps: %{updated_at: updated_at}}) - when is_list(consensus_block_number) do - # ShareLocks order already enforced by `acquire_blocks` (see docs: sharelocks.md) - {_, result} = - repo.update_all( - from(block in Block, where: block.number in ^consensus_block_number, select: block.number), - [set: [consensus: false, updated_at: updated_at]], - timeout: timeout - ) - - {:ok, result} - rescue - postgrex_error in Postgrex.Error -> - {:error, %{exception: postgrex_error, consensus_block_numbers: consensus_block_number}} - end - - defp lose_invalid_neighbour_consensus(repo, where_invalid_neighbour, %{ + defp lose_consensus(repo, hashes, consensus_block_numbers, changes_list, %{ timeout: timeout, timestamps: %{updated_at: updated_at} }) do - # ShareLocks order already enforced by `acquire_blocks` (see docs: sharelocks.md) - {_, result} = + acquire_query = + from( + block in where_invalid_neighbour(changes_list), + or_where: block.number in ^consensus_block_numbers, + # we also need to acquire blocks that will be upserted here, for ordering + or_where: block.hash in ^hashes, + select: block.hash, + # Enforce Block ShareLocks order (see docs: sharelocks.md) + order_by: [asc: block.hash], + lock: "FOR UPDATE" + ) + + {_, removed_consensus_block_hashes} = repo.update_all( - from(block in where_invalid_neighbour, select: block.number), + from( + block in Block, + join: s in subquery(acquire_query), + on: block.hash == s.hash, + # we don't want to remove consensus from blocks that will be upserted + where: block.hash not in ^hashes, + select: block.hash + ), [set: [consensus: false, updated_at: updated_at]], timeout: timeout ) - {:ok, result} + {:ok, removed_consensus_block_hashes} rescue postgrex_error in Postgrex.Error -> - {:error, %{exception: postgrex_error, where_invalid_neighbour: where_invalid_neighbour}} + {:error, %{exception: postgrex_error, consensus_block_numbers: consensus_block_numbers}} end - defp remove_nonconsensus_token_transfers(repo, nonconsensus_block_numbers, %{timeout: timeout}) do + defp remove_nonconsensus_token_transfers(repo, forked_transaction_hashes, %{timeout: timeout}) do ordered_token_transfers = - from(token_transfer in TokenTransfer, - where: token_transfer.block_number in ^nonconsensus_block_numbers, - select: map(token_transfer, [:transaction_hash, :log_index]), + from( + token_transfer in TokenTransfer, + where: token_transfer.transaction_hash in ^forked_transaction_hashes, + select: token_transfer.transaction_hash, # Enforce TokenTransfer ShareLocks order (see docs: sharelocks.md) order_by: [ token_transfer.transaction_hash, @@ -417,91 +366,60 @@ defmodule Explorer.Chain.Import.Runner.Blocks do from(token_transfer in TokenTransfer, select: map(token_transfer, [:transaction_hash, :log_index]), inner_join: ordered_token_transfer in subquery(ordered_token_transfers), - on: - ordered_token_transfer.transaction_hash == - token_transfer.transaction_hash and - ordered_token_transfer.log_index == token_transfer.log_index + on: ordered_token_transfer.transaction_hash == token_transfer.transaction_hash ) - try do - {_count, deleted_token_transfers} = repo.delete_all(query, timeout: timeout) + {_count, deleted_token_transfers} = repo.delete_all(query, timeout: timeout) - {:ok, deleted_token_transfers} - rescue - postgrex_error in Postgrex.Error -> - {:error, %{exception: postgrex_error, block_numbers: nonconsensus_block_numbers}} - end + {:ok, deleted_token_transfers} + rescue + postgrex_error in Postgrex.Error -> + {:error, %{exception: postgrex_error, transactions: forked_transaction_hashes}} end - defp remove_nonconsensus_internal_transactions(repo, nonconsensus_block_numbers, forked_transactions, %{ - timeout: timeout - }) do - forked_transaction_hashes = Enum.map(forked_transactions, & &1.hash) - - transaction_query = - from(transaction in Transaction, - where: transaction.block_number in ^nonconsensus_block_numbers, - or_where: transaction.hash in ^forked_transaction_hashes, - select: map(transaction, [:hash]) - ) - + defp remove_nonconsensus_internal_transactions(repo, forked_transaction_hashes, %{timeout: timeout}) do query = from(internal_transaction in InternalTransaction, - inner_join: transaction in subquery(transaction_query), - on: internal_transaction.transaction_hash == transaction.hash, + where: internal_transaction.transaction_hash in ^forked_transaction_hashes, select: map(internal_transaction, [:transaction_hash, :index]) ) - try do - # ShareLocks order already enforced by `acquire_internal_transactions` (see docs: sharelocks.md) - {_count, deleted_internal_transactions} = repo.delete_all(query, timeout: timeout) + # ShareLocks order already enforced by `acquire_internal_transactions` (see docs: sharelocks.md) + {_count, deleted_internal_transactions} = repo.delete_all(query, timeout: timeout) - {:ok, deleted_internal_transactions} - rescue - postgrex_error in Postgrex.Error -> - {:error, %{exception: postgrex_error, block_numbers: nonconsensus_block_numbers}} - end + {:ok, deleted_internal_transactions} + rescue + postgrex_error in Postgrex.Error -> + {:error, %{exception: postgrex_error, transactions: forked_transaction_hashes}} end - defp remove_nonconsensus_logs(repo, nonconsensus_block_numbers, forked_transactions, %{timeout: timeout}) do - forked_transaction_hashes = Enum.map(forked_transactions, & &1.hash) - - transaction_query = - from(transaction in Transaction, - where: transaction.block_number in ^nonconsensus_block_numbers, - or_where: transaction.hash in ^forked_transaction_hashes, - select: map(transaction, [:hash]), - order_by: transaction.hash - ) - + defp remove_nonconsensus_logs(repo, forked_transaction_hashes, %{timeout: timeout}) do ordered_logs = - from(log in Log, - inner_join: transaction in subquery(transaction_query), - on: log.transaction_hash == transaction.hash, - select: map(log, [:transaction_hash, :index]), + from( + log in Log, + where: log.transaction_hash in ^forked_transaction_hashes, + select: log.transaction_hash, # Enforce Log ShareLocks order (see docs: sharelocks.md) order_by: [ log.transaction_hash, log.index ], - lock: "FOR UPDATE OF l0" + lock: "FOR UPDATE" ) query = from(log in Log, select: map(log, [:transaction_hash, :index]), inner_join: ordered_log in subquery(ordered_logs), - on: ordered_log.transaction_hash == log.transaction_hash and ordered_log.index == log.index + on: ordered_log.transaction_hash == log.transaction_hash ) - try do - {_count, deleted_logs} = repo.delete_all(query, timeout: timeout) + {_count, deleted_logs} = repo.delete_all(query, timeout: timeout) - {:ok, deleted_logs} - rescue - postgrex_error in Postgrex.Error -> - {:error, %{exception: postgrex_error, block_numbers: nonconsensus_block_numbers}} - end + {:ok, deleted_logs} + rescue + postgrex_error in Postgrex.Error -> + {:error, %{exception: postgrex_error, transactions: forked_transaction_hashes}} end defp delete_address_token_balances(_, [], _), do: {:ok, []} diff --git a/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs b/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs index 370fd65021..3cbd7f4834 100644 --- a/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs +++ b/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs @@ -117,10 +117,12 @@ defmodule Explorer.Chain.Import.Runner.BlocksTest do test "remove_nonconsensus_token_transfers deletes token transfer rows with matching block number when new consensus block is inserted", %{consensus_block: %{number: block_number} = block, options: options} do - insert(:block, number: block_number, consensus: true) + consensus_block = insert(:block, number: block_number, consensus: true) + + transaction = insert(:transaction) |> with_block(consensus_block) %TokenTransfer{transaction_hash: transaction_hash, log_index: log_index} = - insert(:token_transfer, block_number: block_number, transaction: insert(:transaction)) + insert(:token_transfer, block_number: block_number, transaction: transaction) assert count(TokenTransfer) == 1 @@ -136,7 +138,11 @@ defmodule Explorer.Chain.Import.Runner.BlocksTest do test "remove_nonconsensus_token_transfers does not delete token transfer rows with matching block number when new consensus block wasn't inserted", %{consensus_block: %{number: block_number} = block, options: options} do - insert(:token_transfer, block_number: block_number, transaction: insert(:transaction)) + consensus_block = insert(:block, number: block_number, consensus: true) + + transaction = insert(:transaction) |> with_block(consensus_block) + + insert(:token_transfer, block_number: block_number, transaction: transaction) count = 1 From bc47fe957e77116f342fa1db37d5a86f1b77abe0 Mon Sep 17 00:00:00 2001 From: pasqu4le Date: Fri, 20 Sep 2019 18:14:05 +0200 Subject: [PATCH 11/15] changelog entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6402a0acd..a525fc5951 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## Current ### Features +- [#2717](https://github.com/poanetwork/blockscout/pull/2717) - Improve speed of nonconsensus data removal - [#2679](https://github.com/poanetwork/blockscout/pull/2679) - added fixed height for card chain blocks and card chain transactions - [#2678](https://github.com/poanetwork/blockscout/pull/2678) - fixed dashboard banner height bug - [#2672](https://github.com/poanetwork/blockscout/pull/2672) - added new theme for xUSDT From a94d2fb37e054690895a0e4f888da6af660ba0b0 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Mon, 23 Sep 2019 13:38:06 +0300 Subject: [PATCH 12/15] fix ci --- .circleci/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index b3abd48786..ba922ddfff 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -39,6 +39,9 @@ jobs: - run: mix deps.get + - run: + command: sed -i '68,68 s/^/%/' ./deps/hackney/src/hackney_ssl.erl + - restore_cache: keys: - v7-npm-install-{{ .Branch }}-{{ checksum "apps/block_scout_web/assets/package-lock.json" }} From 765987979ddaf6fa215ffe477d6315fe42752145 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Mon, 23 Sep 2019 13:48:14 +0300 Subject: [PATCH 13/15] bump cache version --- .circleci/config.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ba922ddfff..a2b372d416 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -33,9 +33,9 @@ jobs: - restore_cache: keys: - - v7-mix-compile-{{ checksum "OTP_VERSION.lock" }}-{{ checksum "ELIXIR_VERSION.lock" }}-{{ checksum "mix.lock" }} - - v7-mix-compile-{{ checksum "OTP_VERSION.lock" }}-{{ checksum "ELIXIR_VERSION.lock" }}-{{ checksum "mix.exs" }} - - v7-mix-compile-{{ checksum "OTP_VERSION.lock" }}-{{ checksum "ELIXIR_VERSION.lock" }} + - v8-mix-compile-{{ checksum "OTP_VERSION.lock" }}-{{ checksum "ELIXIR_VERSION.lock" }}-{{ checksum "mix.lock" }} + - v8-mix-compile-{{ checksum "OTP_VERSION.lock" }}-{{ checksum "ELIXIR_VERSION.lock" }}-{{ checksum "mix.exs" }} + - v8-mix-compile-{{ checksum "OTP_VERSION.lock" }}-{{ checksum "ELIXIR_VERSION.lock" }} - run: mix deps.get @@ -86,17 +86,17 @@ jobs: # `deps` needs to be cached with `_build` because `_build` will symlink into `deps` - save_cache: - key: v7-mix-compile-{{ checksum "OTP_VERSION.lock" }}-{{ checksum "ELIXIR_VERSION.lock" }}-{{ checksum "mix.lock" }} + key: v8-mix-compile-{{ checksum "OTP_VERSION.lock" }}-{{ checksum "ELIXIR_VERSION.lock" }}-{{ checksum "mix.lock" }} paths: - deps - _build - save_cache: - key: v7-mix-compile-{{ checksum "OTP_VERSION.lock" }}-{{ checksum "ELIXIR_VERSION.lock" }}-{{ checksum "mix.exs" }} + key: v8-mix-compile-{{ checksum "OTP_VERSION.lock" }}-{{ checksum "ELIXIR_VERSION.lock" }}-{{ checksum "mix.exs" }} paths: - deps - _build - save_cache: - key: v7-mix-compile-{{ checksum "OTP_VERSION.lock" }}-{{ checksum "ELIXIR_VERSION.lock" }} + key: v8-mix-compile-{{ checksum "OTP_VERSION.lock" }}-{{ checksum "ELIXIR_VERSION.lock" }} paths: - deps - _build From f19814cade66a9f9f192b4ec24d94909d2e63b3c Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Mon, 23 Sep 2019 14:01:22 +0300 Subject: [PATCH 14/15] add CHANGELOG entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6402a0acd..7c0c50d0f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ - [#2619](https://github.com/poanetwork/blockscout/pull/2619) - Enforce DB transaction's order to prevent deadlocks ### Chore +- [#2724](https://github.com/poanetwork/blockscout/pull/2724) - fix ci by commenting a line in hackney library ## 2.0.4-beta From 455d6d647f9e50807e3137c4673dc9a0b5bad204 Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Fri, 20 Sep 2019 15:29:41 +0300 Subject: [PATCH 15/15] Addresses counter --- CHANGELOG.md | 1 + .../controllers/address_controller.ex | 2 +- .../controllers/chain_controller.ex | 2 +- .../lib/block_scout_web/notifier.ex | 2 +- .../channels/address_channel_test.exs | 16 +-- .../controllers/address_controller_test.exs | 10 +- .../api/rpc/address_controller_test.exs | 4 +- .../api/rpc/eth_controller_test.exs | 4 +- .../controllers/chain_controller_test.exs | 8 +- .../features/viewing_addresses_test.exs | 6 +- .../features/viewing_app_test.exs | 6 +- .../features/viewing_chain_test.exs | 34 ++--- apps/explorer/config/config.exs | 5 + apps/explorer/config/test.exs | 2 + apps/explorer/lib/explorer/application.ex | 1 + apps/explorer/lib/explorer/chain.ex | 23 +++- apps/explorer/lib/explorer/chain/address.ex | 10 ++ .../explorer/counters/addresses_counter.ex | 125 ++++++++++++++++++ apps/explorer/test/explorer/chain_test.exs | 17 +++ .../counters/addresses_counter_test.exs | 16 +++ 20 files changed, 246 insertions(+), 48 deletions(-) create mode 100644 apps/explorer/lib/explorer/counters/addresses_counter.ex create mode 100644 apps/explorer/test/explorer/counters/addresses_counter_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index b6402a0acd..5fbc67722c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ - [#2663](https://github.com/poanetwork/blockscout/pull/2663) - Fetch address counters in parallel ### Fixes +- [#2718](https://github.com/poanetwork/blockscout/pull/2718) - Include all addresses taking part in transactions in wallets' addresses counter - [#2709](https://github.com/poanetwork/blockscout/pull/2709) - Fix stuck label and value for uncle block height - [#2707](https://github.com/poanetwork/blockscout/pull/2707) - fix for dashboard banner chart legend items - [#2706](https://github.com/poanetwork/blockscout/pull/2706) - fix empty total_supply in coin gecko response diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/address_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/address_controller.ex index 5916359281..e2cb1f6db7 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/address_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/address_controller.ex @@ -61,7 +61,7 @@ defmodule BlockScoutWeb.AddressController do def index(conn, _params) do render(conn, "index.html", current_path: current_path(conn), - address_count: Chain.count_addresses_with_balance_from_cache() + address_count: Chain.count_addresses_from_cache() ) end diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/chain_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/chain_controller.ex index ba5d547cf0..936d3ab0a0 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/chain_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/chain_controller.ex @@ -28,7 +28,7 @@ defmodule BlockScoutWeb.ChainController do render( conn, "show.html", - address_count: Chain.count_addresses_with_balance_from_cache(), + address_count: Chain.count_addresses_from_cache(), average_block_time: AverageBlockTime.average_block_time(), exchange_rate: exchange_rate, chart_data_path: market_history_chart_path(conn, :show), diff --git a/apps/block_scout_web/lib/block_scout_web/notifier.ex b/apps/block_scout_web/lib/block_scout_web/notifier.ex index 7c07c8524e..5955ca02d4 100644 --- a/apps/block_scout_web/lib/block_scout_web/notifier.ex +++ b/apps/block_scout_web/lib/block_scout_web/notifier.ex @@ -14,7 +14,7 @@ defmodule BlockScoutWeb.Notifier do alias Phoenix.View def handle_event({:chain_event, :addresses, type, addresses}) when type in [:realtime, :on_demand] do - Endpoint.broadcast("addresses:new_address", "count", %{count: Chain.count_addresses_with_balance_from_cache()}) + Endpoint.broadcast("addresses:new_address", "count", %{count: Chain.count_addresses_from_cache()}) addresses |> Stream.reject(fn %Address{fetched_coin_balance: fetched_coin_balance} -> is_nil(fetched_coin_balance) end) diff --git a/apps/block_scout_web/test/block_scout_web/channels/address_channel_test.exs b/apps/block_scout_web/test/block_scout_web/channels/address_channel_test.exs index 49d643fe27..357a73eba8 100644 --- a/apps/block_scout_web/test/block_scout_web/channels/address_channel_test.exs +++ b/apps/block_scout_web/test/block_scout_web/channels/address_channel_test.exs @@ -1,11 +1,11 @@ defmodule BlockScoutWeb.AddressChannelTest do use BlockScoutWeb.ChannelCase, - # ETS tables are shared in `Explorer.Counters.AddressesWithBalanceCounter` + # ETS tables are shared in `Explorer.Counters.AddressesCounter` async: false alias BlockScoutWeb.UserSocket alias BlockScoutWeb.Notifier - alias Explorer.Counters.AddressesWithBalanceCounter + alias Explorer.Counters.AddressesCounter test "subscribed user is notified of new_address count event" do topic = "addresses:new_address" @@ -13,8 +13,8 @@ defmodule BlockScoutWeb.AddressChannelTest do address = insert(:address) - start_supervised!(AddressesWithBalanceCounter) - AddressesWithBalanceCounter.consolidate() + start_supervised!(AddressesCounter) + AddressesCounter.consolidate() Notifier.handle_event({:chain_event, :addresses, :realtime, [address]}) @@ -55,8 +55,8 @@ defmodule BlockScoutWeb.AddressChannelTest do test "notified of balance_update for matching address", %{address: address, topic: topic} do address_with_balance = %{address | fetched_coin_balance: 1} - start_supervised!(AddressesWithBalanceCounter) - AddressesWithBalanceCounter.consolidate() + start_supervised!(AddressesCounter) + AddressesCounter.consolidate() Notifier.handle_event({:chain_event, :addresses, :realtime, [address_with_balance]}) @@ -67,8 +67,8 @@ defmodule BlockScoutWeb.AddressChannelTest do end test "not notified of balance_update if fetched_coin_balance is nil", %{address: address} do - start_supervised!(AddressesWithBalanceCounter) - AddressesWithBalanceCounter.consolidate() + start_supervised!(AddressesCounter) + AddressesCounter.consolidate() Notifier.handle_event({:chain_event, :addresses, :realtime, [address]}) diff --git a/apps/block_scout_web/test/block_scout_web/controllers/address_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/address_controller_test.exs index 7eb0545188..baae81bc1f 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/address_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/address_controller_test.exs @@ -3,7 +3,7 @@ defmodule BlockScoutWeb.AddressControllerTest do # ETS tables are shared in `Explorer.Counters.*` async: false - alias Explorer.Counters.AddressesWithBalanceCounter + alias Explorer.Counters.AddressesCounter describe "GET index/2" do test "returns top addresses", %{conn: conn} do @@ -12,8 +12,8 @@ defmodule BlockScoutWeb.AddressControllerTest do |> Enum.map(&insert(:address, fetched_coin_balance: &1)) |> Enum.map(& &1.hash) - start_supervised!(AddressesWithBalanceCounter) - AddressesWithBalanceCounter.consolidate() + start_supervised!(AddressesCounter) + AddressesCounter.consolidate() conn = get(conn, address_path(conn, :index, %{type: "JSON"})) {:ok, %{"items" => items}} = Poison.decode(conn.resp_body) @@ -25,8 +25,8 @@ defmodule BlockScoutWeb.AddressControllerTest do address = insert(:address, fetched_coin_balance: 1) insert(:address_name, address: address, primary: true, name: "POA Wallet") - start_supervised!(AddressesWithBalanceCounter) - AddressesWithBalanceCounter.consolidate() + start_supervised!(AddressesCounter) + AddressesCounter.consolidate() conn = get(conn, address_path(conn, :index, %{type: "JSON"})) diff --git a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/address_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/address_controller_test.exs index 96716f0935..7e321ac4cf 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/address_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/address_controller_test.exs @@ -6,7 +6,7 @@ defmodule BlockScoutWeb.API.RPC.AddressControllerTest do alias BlockScoutWeb.API.RPC.AddressController alias Explorer.Chain alias Explorer.Chain.{Events.Subscriber, Transaction, Wei} - alias Explorer.Counters.{AddressesWithBalanceCounter, AverageBlockTime} + alias Explorer.Counters.{AddressesCounter, AverageBlockTime} alias Indexer.Fetcher.CoinBalanceOnDemand alias Explorer.Repo @@ -22,7 +22,7 @@ defmodule BlockScoutWeb.API.RPC.AddressControllerTest do start_supervised!({Task.Supervisor, name: Indexer.TaskSupervisor}) start_supervised!(AverageBlockTime) start_supervised!({CoinBalanceOnDemand, [mocked_json_rpc_named_arguments, [name: CoinBalanceOnDemand]]}) - start_supervised!(AddressesWithBalanceCounter) + start_supervised!(AddressesCounter) Application.put_env(:explorer, AverageBlockTime, enabled: true) diff --git a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/eth_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/eth_controller_test.exs index 4b43909ac4..5ab050aefc 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/eth_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/eth_controller_test.exs @@ -1,7 +1,7 @@ defmodule BlockScoutWeb.API.RPC.EthControllerTest do use BlockScoutWeb.ConnCase, async: false - alias Explorer.Counters.{AddressesWithBalanceCounter, AverageBlockTime} + alias Explorer.Counters.{AddressesCounter, AverageBlockTime} alias Explorer.Repo alias Indexer.Fetcher.CoinBalanceOnDemand @@ -14,7 +14,7 @@ defmodule BlockScoutWeb.API.RPC.EthControllerTest do start_supervised!({Task.Supervisor, name: Indexer.TaskSupervisor}) start_supervised!(AverageBlockTime) start_supervised!({CoinBalanceOnDemand, [mocked_json_rpc_named_arguments, [name: CoinBalanceOnDemand]]}) - start_supervised!(AddressesWithBalanceCounter) + start_supervised!(AddressesCounter) Application.put_env(:explorer, AverageBlockTime, enabled: true) diff --git a/apps/block_scout_web/test/block_scout_web/controllers/chain_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/chain_controller_test.exs index fa82bb5bab..def232741c 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/chain_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/chain_controller_test.exs @@ -1,18 +1,18 @@ defmodule BlockScoutWeb.ChainControllerTest do use BlockScoutWeb.ConnCase, - # ETS table is shared in `Explorer.Counters.AddressesWithBalanceCounter` + # ETS table is shared in `Explorer.Counters.AddressesCounter` async: false import BlockScoutWeb.WebRouter.Helpers, only: [chain_path: 2, block_path: 3, transaction_path: 3, address_path: 3] alias Explorer.Chain.Block - alias Explorer.Counters.AddressesWithBalanceCounter + alias Explorer.Counters.AddressesCounter setup do Supervisor.terminate_child(Explorer.Supervisor, Explorer.Chain.Cache.Blocks.child_id()) Supervisor.restart_child(Explorer.Supervisor, Explorer.Chain.Cache.Blocks.child_id()) - start_supervised!(AddressesWithBalanceCounter) - AddressesWithBalanceCounter.consolidate() + start_supervised!(AddressesCounter) + AddressesCounter.consolidate() :ok end diff --git a/apps/block_scout_web/test/block_scout_web/features/viewing_addresses_test.exs b/apps/block_scout_web/test/block_scout_web/features/viewing_addresses_test.exs index b5f17afc49..bb223077a2 100644 --- a/apps/block_scout_web/test/block_scout_web/features/viewing_addresses_test.exs +++ b/apps/block_scout_web/test/block_scout_web/features/viewing_addresses_test.exs @@ -3,7 +3,7 @@ defmodule BlockScoutWeb.ViewingAddressesTest do # Because ETS tables is shared for `Explorer.Counters.*` async: false - alias Explorer.Counters.AddressesWithBalanceCounter + alias Explorer.Counters.AddressesCounter alias BlockScoutWeb.{AddressPage, AddressView, Notifier} setup do @@ -58,8 +58,8 @@ defmodule BlockScoutWeb.ViewingAddressesTest do [first_address | _] = addresses [last_address | _] = Enum.reverse(addresses) - start_supervised!(AddressesWithBalanceCounter) - AddressesWithBalanceCounter.consolidate() + start_supervised!(AddressesCounter) + AddressesCounter.consolidate() session |> AddressPage.visit_page() diff --git a/apps/block_scout_web/test/block_scout_web/features/viewing_app_test.exs b/apps/block_scout_web/test/block_scout_web/features/viewing_app_test.exs index 27742d31eb..ade1f29b15 100644 --- a/apps/block_scout_web/test/block_scout_web/features/viewing_app_test.exs +++ b/apps/block_scout_web/test/block_scout_web/features/viewing_app_test.exs @@ -5,11 +5,11 @@ defmodule BlockScoutWeb.ViewingAppTest do alias BlockScoutWeb.AppPage alias BlockScoutWeb.Counters.BlocksIndexedCounter - alias Explorer.Counters.AddressesWithBalanceCounter + alias Explorer.Counters.AddressesCounter setup do - start_supervised!(AddressesWithBalanceCounter) - AddressesWithBalanceCounter.consolidate() + start_supervised!(AddressesCounter) + AddressesCounter.consolidate() :ok end diff --git a/apps/block_scout_web/test/block_scout_web/features/viewing_chain_test.exs b/apps/block_scout_web/test/block_scout_web/features/viewing_chain_test.exs index 27b4dbb329..367e8b8b7c 100644 --- a/apps/block_scout_web/test/block_scout_web/features/viewing_chain_test.exs +++ b/apps/block_scout_web/test/block_scout_web/features/viewing_chain_test.exs @@ -7,7 +7,7 @@ defmodule BlockScoutWeb.ViewingChainTest do alias BlockScoutWeb.{AddressPage, BlockPage, ChainPage, TransactionPage} alias Explorer.Chain.Block - alias Explorer.Counters.AddressesWithBalanceCounter + alias Explorer.Counters.AddressesCounter setup do Supervisor.terminate_child(Explorer.Supervisor, Explorer.Chain.Cache.Blocks.child_id()) @@ -35,8 +35,8 @@ defmodule BlockScoutWeb.ViewingChainTest do test "search for address", %{session: session} do address = insert(:address) - start_supervised!(AddressesWithBalanceCounter) - AddressesWithBalanceCounter.consolidate() + start_supervised!(AddressesCounter) + AddressesCounter.consolidate() session |> ChainPage.visit_page() @@ -49,8 +49,8 @@ defmodule BlockScoutWeb.ViewingChainTest do test "search for blocks from chain page", %{session: session} do block = insert(:block, number: 6) - start_supervised!(AddressesWithBalanceCounter) - AddressesWithBalanceCounter.consolidate() + start_supervised!(AddressesCounter) + AddressesCounter.consolidate() session |> ChainPage.visit_page() @@ -59,8 +59,8 @@ defmodule BlockScoutWeb.ViewingChainTest do end test "blocks list", %{session: session} do - start_supervised!(AddressesWithBalanceCounter) - AddressesWithBalanceCounter.consolidate() + start_supervised!(AddressesCounter) + AddressesCounter.consolidate() session |> ChainPage.visit_page() @@ -70,8 +70,8 @@ defmodule BlockScoutWeb.ViewingChainTest do test "inserts place holder blocks on render for out of order blocks", %{session: session} do insert(:block, number: 409) - start_supervised!(AddressesWithBalanceCounter) - AddressesWithBalanceCounter.consolidate() + start_supervised!(AddressesCounter) + AddressesCounter.consolidate() session |> ChainPage.visit_page() @@ -84,8 +84,8 @@ defmodule BlockScoutWeb.ViewingChainTest do test "search for transactions", %{session: session} do transaction = insert(:transaction) - start_supervised!(AddressesWithBalanceCounter) - AddressesWithBalanceCounter.consolidate() + start_supervised!(AddressesCounter) + AddressesCounter.consolidate() session |> ChainPage.visit_page() @@ -94,8 +94,8 @@ defmodule BlockScoutWeb.ViewingChainTest do end test "transactions list", %{session: session} do - start_supervised!(AddressesWithBalanceCounter) - AddressesWithBalanceCounter.consolidate() + start_supervised!(AddressesCounter) + AddressesCounter.consolidate() session |> ChainPage.visit_page() @@ -111,8 +111,8 @@ defmodule BlockScoutWeb.ViewingChainTest do |> with_contract_creation(contract_address) |> with_block(block) - start_supervised!(AddressesWithBalanceCounter) - AddressesWithBalanceCounter.consolidate() + start_supervised!(AddressesCounter) + AddressesCounter.consolidate() session |> ChainPage.visit_page() @@ -138,8 +138,8 @@ defmodule BlockScoutWeb.ViewingChainTest do token_contract_address: contract_token_address ) - start_supervised!(AddressesWithBalanceCounter) - AddressesWithBalanceCounter.consolidate() + start_supervised!(AddressesCounter) + AddressesCounter.consolidate() session |> ChainPage.visit_page() diff --git a/apps/explorer/config/config.exs b/apps/explorer/config/config.exs index 48d9b55610..0cf1a05b53 100644 --- a/apps/explorer/config/config.exs +++ b/apps/explorer/config/config.exs @@ -47,6 +47,11 @@ balances_update_interval = end config :explorer, Explorer.Counters.AddressesWithBalanceCounter, + enabled: false, + enable_consolidation: true, + update_interval_in_seconds: balances_update_interval || 30 * 60 + +config :explorer, Explorer.Counters.AddressesCounter, enabled: true, enable_consolidation: true, update_interval_in_seconds: balances_update_interval || 30 * 60 diff --git a/apps/explorer/config/test.exs b/apps/explorer/config/test.exs index d91f4dbdfc..8530014cd8 100644 --- a/apps/explorer/config/test.exs +++ b/apps/explorer/config/test.exs @@ -21,6 +21,8 @@ config :explorer, Explorer.Counters.AverageBlockTime, enabled: false config :explorer, Explorer.Counters.AddressesWithBalanceCounter, enabled: false, enable_consolidation: false +config :explorer, Explorer.Counters.AddressesCounter, enabled: false, enable_consolidation: false + config :explorer, Explorer.Market.History.Cataloger, enabled: false config :explorer, Explorer.Tracer, disabled?: false diff --git a/apps/explorer/lib/explorer/application.ex b/apps/explorer/lib/explorer/application.ex index 9645572b62..53d8c03ea3 100644 --- a/apps/explorer/lib/explorer/application.ex +++ b/apps/explorer/lib/explorer/application.ex @@ -68,6 +68,7 @@ defmodule Explorer.Application do configure(Explorer.KnownTokens), configure(Explorer.Market.History.Cataloger), configure(Explorer.Counters.AddressesWithBalanceCounter), + configure(Explorer.Counters.AddressesCounter), configure(Explorer.Counters.AverageBlockTime), configure(Explorer.Validator.MetadataProcessor), configure(Explorer.Staking.EpochCounter) diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index 36cfae424e..f89d933d6d 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -58,7 +58,7 @@ defmodule Explorer.Chain do } alias Explorer.Chain.Import.Runner - alias Explorer.Counters.AddressesWithBalanceCounter + alias Explorer.Counters.{AddressesCounter, AddressesWithBalanceCounter} alias Explorer.Market.MarketHistoryCache alias Explorer.{PagingOptions, Repo} @@ -118,6 +118,14 @@ defmodule Explorer.Chain do AddressesWithBalanceCounter.fetch() end + @doc """ + Gets from the cache the count of all `t:Explorer.Chain.Address.t/0`'s + """ + @spec count_addresses_from_cache :: non_neg_integer() + def count_addresses_from_cache do + AddressesCounter.fetch() + end + @doc """ Counts the number of addresses with fetched coin balance > 0. @@ -131,6 +139,19 @@ defmodule Explorer.Chain do ) end + @doc """ + Counts the number of all addresses. + + This function should be used with caution. In larger databases, it may take a + while to have the return back. + """ + def count_addresses do + Repo.one( + Address.count(), + timeout: :infinity + ) + end + @doc """ `t:Explorer.Chain.InternalTransaction/0`s from the address with the given `hash`. diff --git a/apps/explorer/lib/explorer/chain/address.ex b/apps/explorer/lib/explorer/chain/address.ex index 96d4dc7ab4..76ea5e85dd 100644 --- a/apps/explorer/lib/explorer/chain/address.ex +++ b/apps/explorer/lib/explorer/chain/address.ex @@ -237,6 +237,16 @@ defmodule Explorer.Chain.Address do ) end + @doc """ + Counts all the addresses. + """ + def count do + from( + a in Address, + select: fragment("COUNT(*)") + ) + end + defimpl String.Chars do @doc """ Uses `hash` as string representation, formatting it according to the eip-55 specification diff --git a/apps/explorer/lib/explorer/counters/addresses_counter.ex b/apps/explorer/lib/explorer/counters/addresses_counter.ex new file mode 100644 index 0000000000..5febd8fbb9 --- /dev/null +++ b/apps/explorer/lib/explorer/counters/addresses_counter.ex @@ -0,0 +1,125 @@ +defmodule Explorer.Counters.AddressesCounter do + @moduledoc """ + Caches the number of all addresses. + + It loads the count asynchronously and in a time interval of 30 minutes. + """ + + use GenServer + + alias Explorer.Chain + + @table :addresses_counter + + @cache_key "addresses" + + def table_name do + @table + end + + def cache_key do + @cache_key + end + + # It is undesirable to automatically start the consolidation in all environments. + # Consider the test environment: if the consolidation initiates but does not + # finish before a test ends, that test will fail. This way, hundreds of + # tests were failing before disabling the consolidation and the scheduler in + # the test env. + config = Application.get_env(:explorer, Explorer.Counters.AddressesCounter) + @enable_consolidation Keyword.get(config, :enable_consolidation) + + @update_interval_in_seconds Keyword.get(config, :update_interval_in_seconds) + + @doc """ + Starts a process to periodically update the counter of the token holders. + """ + @spec start_link(term()) :: GenServer.on_start() + def start_link(_) do + GenServer.start_link(__MODULE__, :ok, name: __MODULE__) + end + + @impl true + def init(_args) do + create_table() + + {:ok, %{consolidate?: enable_consolidation?()}, {:continue, :ok}} + end + + def create_table do + opts = [ + :set, + :named_table, + :public, + read_concurrency: true + ] + + :ets.new(table_name(), opts) + end + + defp schedule_next_consolidation do + Process.send_after(self(), :consolidate, :timer.seconds(@update_interval_in_seconds)) + end + + @doc """ + Inserts new items into the `:ets` table. + """ + def insert_counter({key, info}) do + :ets.insert(table_name(), {key, info}) + end + + @impl true + def handle_continue(:ok, %{consolidate?: true} = state) do + consolidate() + schedule_next_consolidation() + + {:noreply, state} + end + + @impl true + def handle_continue(:ok, state) do + {:noreply, state} + end + + @impl true + def handle_info(:consolidate, state) do + consolidate() + + schedule_next_consolidation() + + {:noreply, state} + end + + @doc """ + Fetches the info for a specific item from the `:ets` table. + """ + def fetch do + do_fetch(:ets.lookup(table_name(), cache_key())) + end + + defp do_fetch([{_, result}]), do: result + defp do_fetch([]), do: 0 + + @doc """ + Consolidates the info by populating the `:ets` table with the current database information. + """ + def consolidate do + counter = Chain.count_addresses() + + insert_counter({cache_key(), counter}) + end + + @doc """ + Returns a boolean that indicates whether consolidation is enabled + + In order to choose whether or not to enable the scheduler and the initial + consolidation, change the following Explorer config: + + `config :explorer, Explorer.Counters.AddressesCounter, enable_consolidation: true` + + to: + + `config :explorer, Explorer.Counters.AddressesCounter, enable_consolidation: false` + """ + def enable_consolidation?, do: @enable_consolidation +end diff --git a/apps/explorer/test/explorer/chain_test.exs b/apps/explorer/test/explorer/chain_test.exs index 3ac60b6963..872a3bc5c0 100644 --- a/apps/explorer/test/explorer/chain_test.exs +++ b/apps/explorer/test/explorer/chain_test.exs @@ -27,6 +27,7 @@ defmodule Explorer.ChainTest do alias Explorer.Chain.Supply.ProofOfAuthority alias Explorer.Counters.AddressesWithBalanceCounter + alias Explorer.Counters.AddressesCounter doctest Explorer.Chain @@ -50,6 +51,22 @@ defmodule Explorer.ChainTest do end end + describe "count_addresses_from_cache/0" do + test "returns the number of all addresses" do + insert(:address, fetched_coin_balance: 0) + insert(:address, fetched_coin_balance: 1) + insert(:address, fetched_coin_balance: 2) + + start_supervised!(AddressesCounter) + AddressesCounter.consolidate() + + addresses_with_balance = Chain.count_addresses_from_cache() + + assert is_integer(addresses_with_balance) + assert addresses_with_balance == 3 + end + end + describe "last_db_block_status/0" do test "return no_blocks errors if db is empty" do assert {:error, :no_blocks} = Chain.last_db_block_status() diff --git a/apps/explorer/test/explorer/counters/addresses_counter_test.exs b/apps/explorer/test/explorer/counters/addresses_counter_test.exs new file mode 100644 index 0000000000..b187fd8c0e --- /dev/null +++ b/apps/explorer/test/explorer/counters/addresses_counter_test.exs @@ -0,0 +1,16 @@ +defmodule Explorer.Counters.AddressesCounterTest do + use Explorer.DataCase + + alias Explorer.Counters.AddressesCounter + + test "populates the cache with the number of all addresses" do + insert(:address, fetched_coin_balance: 0) + insert(:address, fetched_coin_balance: 1) + insert(:address, fetched_coin_balance: 2) + + start_supervised!(AddressesCounter) + AddressesCounter.consolidate() + + assert AddressesCounter.fetch() == 3 + end +end