From 326b70fa5289fbdab484049b12ac33407d52e931 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 3 Sep 2019 11:48:28 +0300 Subject: [PATCH 1/8] set correct last calue for coin balances chart data Sometimes the last value for coin history chart data is incorrect because when selecting records they are grouped by day and max value is selected in that day. So if there are a couple of coin balance changes only max value is shown. This PR always shows the latest value from the database by block number. --- apps/explorer/lib/explorer/chain.ex | 7 +++++++ .../lib/explorer/chain/address/coin_balance.ex | 2 +- apps/explorer/test/explorer/chain_test.exs | 16 ++++++++++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index 1581bd4846..b2a7c6fdb1 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -3002,9 +3002,16 @@ defmodule Explorer.Chain do address_hash |> CoinBalance.balances_by_day(latest_block_timestamp) |> Repo.all() + |> replace_last_value(latest_block_timestamp) |> normalize_balances_by_day() end + defp replace_last_value(items, %{value: value, timestamp: timestamp}) do + List.replace_at(items, -1, %{date: Date.convert!(timestamp, Calendar.ISO), value: value}) + end + + defp replace_last_value(items, _), do: items + defp normalize_balances_by_day(balances_by_day) do result = balances_by_day diff --git a/apps/explorer/lib/explorer/chain/address/coin_balance.ex b/apps/explorer/lib/explorer/chain/address/coin_balance.ex index 537f6a0483..f3647a8f54 100644 --- a/apps/explorer/lib/explorer/chain/address/coin_balance.ex +++ b/apps/explorer/lib/explorer/chain/address/coin_balance.ex @@ -112,7 +112,7 @@ defmodule Explorer.Chain.Address.CoinBalance do |> join(:inner, [cb], b in Block, on: cb.block_number == b.number) |> where([cb], cb.address_hash == ^address_hash) |> last(:block_number) - |> select([cb, b], %{timestamp: b.timestamp}) + |> select([cb, b], %{timestamp: b.timestamp, value: cb.value}) end def changeset(%__MODULE__{} = balance, params) do diff --git a/apps/explorer/test/explorer/chain_test.exs b/apps/explorer/test/explorer/chain_test.exs index 187d503621..d9bfec58f4 100644 --- a/apps/explorer/test/explorer/chain_test.exs +++ b/apps/explorer/test/explorer/chain_test.exs @@ -3908,6 +3908,22 @@ defmodule Explorer.ChainTest do %{date: today |> NaiveDateTime.to_date() |> Date.to_string(), value: Decimal.new("1E-15")} ] end + + test "uses last block value if there a couple of change in the same day" do + address = insert(:address) + today = NaiveDateTime.utc_now() + past = Timex.shift(today, hours: -1) + + block_now = insert(:block, timestamp: today) + insert(:fetched_balance, address_hash: address.hash, value: 1, block_number: block_now.number) + + block_past = insert(:block, timestamp: past) + insert(:fetched_balance, address_hash: address.hash, value: 0, block_number: block_past.number) + + [balance] = Chain.address_to_balances_by_day(address.hash) + + assert balance.value == Decimal.new(0) + end end describe "block_combined_rewards/1" do From bf23cc0ec17d0e99bfbccd9c06cc3450a196a9f0 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 3 Sep 2019 11:54:28 +0300 Subject: [PATCH 2/8] add CHANGELOG entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca60257372..8ccd24aa31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - [#2497](https://github.com/poanetwork/blockscout/pull/2497) - Add generic Ordered Cache behaviour and implementation ### Fixes +- [#2660](https://github.com/poanetwork/blockscout/pull/2660) - set correct last value for coin balances chart data - [#2468](https://github.com/poanetwork/blockscout/pull/2468) - fix confirmations for non consensus blocks - [#2610](https://github.com/poanetwork/blockscout/pull/2610) - use CoinGecko instead of CoinMarketcap for exchange rates - [#2640](https://github.com/poanetwork/blockscout/pull/2640) - SVG network icons From acd7fb53b31200f6995a2bfad67f3c3a9286c05e Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 3 Sep 2019 11:58:11 +0300 Subject: [PATCH 3/8] fix test --- apps/explorer/test/explorer/chain_test.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/explorer/test/explorer/chain_test.exs b/apps/explorer/test/explorer/chain_test.exs index d9bfec58f4..d673336ef6 100644 --- a/apps/explorer/test/explorer/chain_test.exs +++ b/apps/explorer/test/explorer/chain_test.exs @@ -3914,10 +3914,10 @@ defmodule Explorer.ChainTest do today = NaiveDateTime.utc_now() past = Timex.shift(today, hours: -1) - block_now = insert(:block, timestamp: today) + block_now = insert(:block, timestamp: today, number: 1) insert(:fetched_balance, address_hash: address.hash, value: 1, block_number: block_now.number) - block_past = insert(:block, timestamp: past) + block_past = insert(:block, timestamp: past, number: 2) insert(:fetched_balance, address_hash: address.hash, value: 0, block_number: block_past.number) [balance] = Chain.address_to_balances_by_day(address.hash) From 05a00b4cc90eb981e2dd118e22abc5635e3ce8ce Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 3 Sep 2019 12:25:57 +0300 Subject: [PATCH 4/8] fix tests --- .../address_coin_balance_by_day_controller_test.exs | 4 ++-- apps/explorer/test/explorer/chain_test.exs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/block_scout_web/test/block_scout_web/controllers/address_coin_balance_by_day_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/address_coin_balance_by_day_controller_test.exs index 1496bb24e9..df714247c2 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/address_coin_balance_by_day_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/address_coin_balance_by_day_controller_test.exs @@ -5,8 +5,8 @@ defmodule BlockScoutWeb.AddressCoinBalanceByDayControllerTest do test "returns the coin balance history grouped by date", %{conn: conn} do address = insert(:address) noon = Timex.now() |> Timex.beginning_of_day() |> Timex.set(hour: 12) - block = insert(:block, timestamp: noon) - block_one_day_ago = insert(:block, timestamp: Timex.shift(noon, days: -1)) + block = insert(:block, timestamp: noon, number: 2) + block_one_day_ago = insert(:block, timestamp: Timex.shift(noon, days: -1), number: 1) insert(:fetched_balance, address_hash: address.hash, value: 1000, block_number: block.number) insert(:fetched_balance, address_hash: address.hash, value: 2000, block_number: block_one_day_ago.number) diff --git a/apps/explorer/test/explorer/chain_test.exs b/apps/explorer/test/explorer/chain_test.exs index d673336ef6..3ac60b6963 100644 --- a/apps/explorer/test/explorer/chain_test.exs +++ b/apps/explorer/test/explorer/chain_test.exs @@ -3879,9 +3879,9 @@ defmodule Explorer.ChainTest do address = insert(:address) today = NaiveDateTime.utc_now() noon = Timex.set(today, hour: 12) - block = insert(:block, timestamp: noon) + block = insert(:block, timestamp: noon, number: 50) yesterday = Timex.shift(noon, days: -1) - block_one_day_ago = insert(:block, timestamp: yesterday) + block_one_day_ago = insert(:block, timestamp: yesterday, number: 49) insert(:fetched_balance, address_hash: address.hash, value: 1000, block_number: block.number) insert(:fetched_balance, address_hash: address.hash, value: 2000, block_number: block_one_day_ago.number) From 5e0b5308b86f59daab0c8595b8dccc6076c279d3 Mon Sep 17 00:00:00 2001 From: Yegor San Date: Wed, 18 Sep 2019 15:27:47 +0300 Subject: [PATCH 5/8] canvas --- .../assets/css/components/_dashboard-banner.scss | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/block_scout_web/assets/css/components/_dashboard-banner.scss b/apps/block_scout_web/assets/css/components/_dashboard-banner.scss index 0cf8071caf..bae39d46a1 100644 --- a/apps/block_scout_web/assets/css/components/_dashboard-banner.scss +++ b/apps/block_scout_web/assets/css/components/_dashboard-banner.scss @@ -72,7 +72,7 @@ $dashboard-banner-chart-axis-font-color: $dashboard-stats-item-value-color !defa } .dashboard-banner-chart-legend { - display: flex; + display: grid; grid-template-columns: 1fr 1fr; padding-bottom: 12px; @@ -81,7 +81,7 @@ $dashboard-banner-chart-axis-font-color: $dashboard-stats-item-value-color !defa padding-left: 12px; padding-top: 3px; position: relative; - padding-right: 60px; + padding-right: 12px; @include media-breakpoint-down(md) { display: flex; From 55339cb1c10e05ece1e3c16732490855287ad957 Mon Sep 17 00:00:00 2001 From: Yegor San Date: Wed, 18 Sep 2019 15:49:03 +0300 Subject: [PATCH 6/8] Updated Changelog.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f794cf3f8..04fb795636 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - [#2663](https://github.com/poanetwork/blockscout/pull/2663) - Fetch address counters in parallel ### Fixes +- [#2707](https://github.com/poanetwork/blockscout/pull/2707) - fix for dashboard banner chart legend items - [#2701](https://github.com/poanetwork/blockscout/pull/2701) - Exclude nonconsensus blocks from avg block time calculation by default - [#2696](https://github.com/poanetwork/blockscout/pull/2696) - do not update fetched_coin_balance with nil - [#2693](https://github.com/poanetwork/blockscout/pull/2693) - remove non consensus internal transactions From af7782f2fe46983159f9e8e00966e57cb1721e8f Mon Sep 17 00:00:00 2001 From: pasqu4le Date: Wed, 4 Sep 2019 18:29:57 +0200 Subject: [PATCH 7/8] Add ETS-based cache for accounts page --- CHANGELOG.md | 1 + .../block_scout_web/test/support/conn_case.ex | 2 + .../test/support/feature_case.ex | 2 + apps/explorer/config/config.exs | 4 + apps/explorer/lib/explorer/application.ex | 4 +- apps/explorer/lib/explorer/chain.ex | 31 ++++++++ .../lib/explorer/chain/cache/accounts.ex | 73 +++++++++++++++++++ .../explorer/chain/cache/accounts_test.exs | 42 +++++++++++ apps/explorer/test/support/data_case.ex | 2 + apps/indexer/lib/indexer/block/fetcher.ex | 5 +- .../lib/indexer/block/realtime/fetcher.ex | 3 + .../lib/indexer/fetcher/block_reward.ex | 5 +- .../lib/indexer/fetcher/coin_balance.ex | 5 +- .../indexer/fetcher/coin_balance_on_demand.ex | 8 +- .../lib/indexer/fetcher/contract_code.ex | 4 +- .../indexer/fetcher/internal_transaction.ex | 3 + .../indexer/fetcher/pending_transaction.ex | 4 +- .../lib/indexer/fetcher/uncle_block.ex | 4 +- 18 files changed, 193 insertions(+), 9 deletions(-) create mode 100644 apps/explorer/lib/explorer/chain/cache/accounts.ex create mode 100644 apps/explorer/test/explorer/chain/cache/accounts_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index f3a0f91771..53299264df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## Current ### Features +- [#2667](https://github.com/poanetwork/blockscout/pull/2667) - Add ETS-based cache for accounts page - [#2679](https://github.com/poanetwork/blockscout/pull/2679) - added fixed height for card chain blocks and card chain transactions - [#2678](https://github.com/poanetwork/blockscout/pull/2678) - fixed dashboard banner height bug - [#2672](https://github.com/poanetwork/blockscout/pull/2672) - added new theme for xUSDT diff --git a/apps/block_scout_web/test/support/conn_case.ex b/apps/block_scout_web/test/support/conn_case.ex index 4ab8f3bcd9..ed3f094c34 100644 --- a/apps/block_scout_web/test/support/conn_case.ex +++ b/apps/block_scout_web/test/support/conn_case.ex @@ -42,6 +42,8 @@ defmodule BlockScoutWeb.ConnCase do Supervisor.terminate_child(Explorer.Supervisor, Explorer.Chain.Cache.Transactions.child_id()) Supervisor.restart_child(Explorer.Supervisor, Explorer.Chain.Cache.Transactions.child_id()) + Supervisor.terminate_child(Explorer.Supervisor, Explorer.Chain.Cache.Accounts.child_id()) + Supervisor.restart_child(Explorer.Supervisor, Explorer.Chain.Cache.Accounts.child_id()) {:ok, conn: Phoenix.ConnTest.build_conn()} end diff --git a/apps/block_scout_web/test/support/feature_case.ex b/apps/block_scout_web/test/support/feature_case.ex index cdacd547bf..f6b1aedf24 100644 --- a/apps/block_scout_web/test/support/feature_case.ex +++ b/apps/block_scout_web/test/support/feature_case.ex @@ -29,6 +29,8 @@ defmodule BlockScoutWeb.FeatureCase do Supervisor.terminate_child(Explorer.Supervisor, Explorer.Chain.Cache.Transactions.child_id()) Supervisor.restart_child(Explorer.Supervisor, Explorer.Chain.Cache.Transactions.child_id()) + Supervisor.terminate_child(Explorer.Supervisor, Explorer.Chain.Cache.Accounts.child_id()) + Supervisor.restart_child(Explorer.Supervisor, Explorer.Chain.Cache.Accounts.child_id()) metadata = Phoenix.Ecto.SQL.Sandbox.metadata_for(Explorer.Repo, self()) {:ok, session} = Wallaby.start_session(metadata: metadata) diff --git a/apps/explorer/config/config.exs b/apps/explorer/config/config.exs index 8a264eaf33..48d9b55610 100644 --- a/apps/explorer/config/config.exs +++ b/apps/explorer/config/config.exs @@ -136,6 +136,10 @@ config :explorer, Explorer.Chain.Cache.Transactions, ttl_check_interval: if(System.get_env("DISABLE_INDEXER") == "true", do: :timer.seconds(1), else: false), global_ttl: if(System.get_env("DISABLE_INDEXER") == "true", do: :timer.seconds(5)) +config :explorer, Explorer.Chain.Cache.Accounts, + ttl_check_interval: if(System.get_env("DISABLE_INDEXER") == "true", do: :timer.seconds(1), else: false), + global_ttl: if(System.get_env("DISABLE_INDEXER") == "true", do: :timer.seconds(5)) + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" diff --git a/apps/explorer/lib/explorer/application.ex b/apps/explorer/lib/explorer/application.ex index 7c55d28f76..9645572b62 100644 --- a/apps/explorer/lib/explorer/application.ex +++ b/apps/explorer/lib/explorer/application.ex @@ -8,6 +8,7 @@ defmodule Explorer.Application do alias Explorer.Admin alias Explorer.Chain.Cache.{ + Accounts, BlockCount, BlockNumber, Blocks, @@ -49,7 +50,8 @@ defmodule Explorer.Application do BlockNumber, con_cache_child_spec(MarketHistoryCache.cache_name()), con_cache_child_spec(RSK.cache_name(), ttl_check_interval: :timer.minutes(1), global_ttl: :timer.minutes(30)), - Transactions + Transactions, + Accounts ] children = base_children ++ configurable_children() diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index 6de1adc9ff..d39945034d 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -49,6 +49,7 @@ defmodule Explorer.Chain do alias Explorer.Chain.Block.{EmissionReward, Reward} alias Explorer.Chain.Cache.{ + Accounts, BlockCount, BlockNumber, Blocks, @@ -1379,6 +1380,36 @@ defmodule Explorer.Chain do def list_top_addresses(options \\ []) do paging_options = Keyword.get(options, :paging_options, @default_paging_options) + if is_nil(paging_options.key) do + paging_options.page_size + |> Accounts.take_enough() + |> case do + nil -> + accounts_with_n = fetch_top_addresses(paging_options) + + accounts_with_n + |> Enum.map(fn {address, _n} -> address end) + |> Accounts.update() + + accounts_with_n + + accounts -> + Enum.map( + accounts, + &{&1, + if is_nil(&1.nonce) do + 0 + else + &1.nonce + 1 + end} + ) + end + else + fetch_top_addresses(paging_options) + end + end + + defp fetch_top_addresses(paging_options) do base_query = from(a in Address, where: a.fetched_coin_balance > ^0, diff --git a/apps/explorer/lib/explorer/chain/cache/accounts.ex b/apps/explorer/lib/explorer/chain/cache/accounts.ex new file mode 100644 index 0000000000..523bd44877 --- /dev/null +++ b/apps/explorer/lib/explorer/chain/cache/accounts.ex @@ -0,0 +1,73 @@ +defmodule Explorer.Chain.Cache.Accounts do + @moduledoc """ + Caches the top Addresses + """ + + alias Explorer.Chain.Address + + use Explorer.Chain.OrderedCache, + name: :accounts, + max_size: 51, + preload: :names, + ttl_check_interval: Application.get_env(:explorer, __MODULE__)[:ttl_check_interval], + global_ttl: Application.get_env(:explorer, __MODULE__)[:global_ttl] + + @type element :: Address.t() + + @type id :: {non_neg_integer(), non_neg_integer()} + + def element_to_id(%Address{fetched_coin_balance: fetched_coin_balance, hash: hash}) do + {fetched_coin_balance, hash} + end + + def prevails?({fetched_coin_balance_a, hash_a}, {fetched_coin_balance_b, hash_b}) do + # same as a query's `order_by: [desc: :fetched_coin_balance, asc: :hash]` + if fetched_coin_balance_a == fetched_coin_balance_b do + hash_a < hash_b + else + fetched_coin_balance_a > fetched_coin_balance_b + end + end + + def drop(nil), do: :ok + + def drop([]), do: :ok + + def drop(addresses) when is_list(addresses) do + # This has to be used by the Indexer insead of `update`. + # The reason being that addresses already in the cache can change their balance + # value and removing or updating them will result into a potentially invalid + # cache status, that would not even get corrected with time. + # The only thing we can safely do when an address in the cache changes its + # `fetched_coin_balance` is to invalidate the whole cache and wait for it + # to be filled again (by the query that it takes the place of when full). + ConCache.update(cache_name(), ids_list_key(), fn ids -> + if drop_needed?(ids, addresses) do + # Remove the addresses immediately + Enum.each(ids, &ConCache.delete(cache_name(), &1)) + + {:ok, []} + else + {:ok, ids} + end + end) + end + + def drop(address), do: drop([address]) + + defp drop_needed?(ids, _addresses) when is_nil(ids), do: false + + defp drop_needed?([], _addresses), do: false + + defp drop_needed?(ids, addresses) do + ids_map = Map.new(ids, fn {balance, hash} -> {hash, balance} end) + + # Result it `true` only when the address is present in the cache already, + # but with a different `fetched_coin_balance` + Enum.find_value(addresses, false, fn address -> + stored_address_balance = Map.get(ids_map, address.hash) + + stored_address_balance && stored_address_balance != address.fetched_coin_balance + end) + end +end diff --git a/apps/explorer/test/explorer/chain/cache/accounts_test.exs b/apps/explorer/test/explorer/chain/cache/accounts_test.exs new file mode 100644 index 0000000000..a1a49bd67b --- /dev/null +++ b/apps/explorer/test/explorer/chain/cache/accounts_test.exs @@ -0,0 +1,42 @@ +defmodule Explorer.Chain.Cache.AccountsTest do + use Explorer.DataCase + + alias Explorer.Chain.Cache.Accounts + alias Explorer.Repo + + describe "drop/1" do + test "does not drop the cache if the address fetched_coin_balance has not changed" do + address = + insert(:address, fetched_coin_balance: 100_000, fetched_coin_balance_block_number: 1) + |> preload_names() + + Accounts.update(address) + + assert Accounts.take(1) == [address] + + Accounts.drop(address) + + assert Accounts.take(1) == [address] + end + + test "drops the cache if an address was in the cache with a different fetched_coin_balance" do + address = + insert(:address, fetched_coin_balance: 100_000, fetched_coin_balance_block_number: 1) + |> preload_names() + + Accounts.update(address) + + assert Accounts.take(1) == [address] + + updated_address = %{address | fetched_coin_balance: 100_001} + + Accounts.drop(updated_address) + + assert Accounts.take(1) == [] + end + end + + defp preload_names(address) do + Repo.preload(address, [:names]) + end +end diff --git a/apps/explorer/test/support/data_case.ex b/apps/explorer/test/support/data_case.ex index 47c15bbca0..e12dd24a82 100644 --- a/apps/explorer/test/support/data_case.ex +++ b/apps/explorer/test/support/data_case.ex @@ -45,6 +45,8 @@ defmodule Explorer.DataCase do Supervisor.restart_child(Explorer.Supervisor, Explorer.Chain.Cache.Blocks.child_id()) Supervisor.terminate_child(Explorer.Supervisor, Explorer.Chain.Cache.Transactions.child_id()) Supervisor.restart_child(Explorer.Supervisor, Explorer.Chain.Cache.Transactions.child_id()) + Supervisor.terminate_child(Explorer.Supervisor, Explorer.Chain.Cache.Accounts.child_id()) + Supervisor.restart_child(Explorer.Supervisor, Explorer.Chain.Cache.Accounts.child_id()) :ok end diff --git a/apps/indexer/lib/indexer/block/fetcher.ex b/apps/indexer/lib/indexer/block/fetcher.ex index 28002f010a..1b95d84086 100644 --- a/apps/indexer/lib/indexer/block/fetcher.ex +++ b/apps/indexer/lib/indexer/block/fetcher.ex @@ -13,7 +13,7 @@ defmodule Indexer.Block.Fetcher do alias Explorer.Chain alias Explorer.Chain.{Address, Block, Hash, Import, Transaction} alias Explorer.Chain.Cache.Blocks, as: BlocksCache - alias Explorer.Chain.Cache.{BlockNumber, Transactions} + alias Explorer.Chain.Cache.{Accounts, BlockNumber, Transactions} alias Indexer.Block.Fetcher.Receipts alias Indexer.Fetcher.{ @@ -176,6 +176,7 @@ defmodule Indexer.Block.Fetcher do result = {:ok, %{inserted: inserted, errors: blocks_errors}} update_block_cache(inserted[:blocks]) update_transactions_cache(inserted[:transactions]) + update_addresses_cache(inserted[:addresses]) result else {step, {:error, reason}} -> {:error, {step, reason}} @@ -197,6 +198,8 @@ defmodule Indexer.Block.Fetcher do Transactions.update(transactions) end + defp update_addresses_cache(addresses), do: Accounts.drop(addresses) + def import( %__MODULE__{broadcast: broadcast, callback_module: callback_module} = state, options diff --git a/apps/indexer/lib/indexer/block/realtime/fetcher.ex b/apps/indexer/lib/indexer/block/realtime/fetcher.ex index 8edd87738d..897eab2cff 100644 --- a/apps/indexer/lib/indexer/block/realtime/fetcher.ex +++ b/apps/indexer/lib/indexer/block/realtime/fetcher.ex @@ -27,6 +27,7 @@ defmodule Indexer.Block.Realtime.Fetcher do alias Ecto.Changeset alias EthereumJSONRPC.{FetchedBalances, Subscription} alias Explorer.Chain + alias Explorer.Chain.Cache.Accounts alias Explorer.Counters.AverageBlockTime alias Indexer.{Block, Tracer} alias Indexer.Block.Realtime.TaskSupervisor @@ -197,6 +198,8 @@ defmodule Indexer.Block.Realtime.Fetcher do json_rpc_named_arguments ) + Accounts.drop(imported[:addresses]) + ok end end diff --git a/apps/indexer/lib/indexer/fetcher/block_reward.ex b/apps/indexer/lib/indexer/fetcher/block_reward.ex index 4a59eb1cbe..ed593bb655 100644 --- a/apps/indexer/lib/indexer/fetcher/block_reward.ex +++ b/apps/indexer/lib/indexer/fetcher/block_reward.ex @@ -17,6 +17,7 @@ defmodule Indexer.Fetcher.BlockReward do alias EthereumJSONRPC.FetchedBeneficiaries alias Explorer.Chain alias Explorer.Chain.{Block, Wei} + alias Explorer.Chain.Cache.Accounts alias Indexer.{BufferedTask, Tracer} alias Indexer.Fetcher.BlockReward.Supervisor, as: BlockRewardSupervisor alias Indexer.Fetcher.CoinBalance @@ -130,7 +131,9 @@ defmodule Indexer.Fetcher.BlockReward do |> add_gas_payments() |> import_block_reward_params() |> case do - {:ok, %{address_coin_balances: address_coin_balances}} -> + {:ok, %{address_coin_balances: address_coin_balances, addresses: addresses}} -> + Accounts.drop(addresses) + CoinBalance.async_fetch_balances(address_coin_balances) retry_errors(errors) diff --git a/apps/indexer/lib/indexer/fetcher/coin_balance.ex b/apps/indexer/lib/indexer/fetcher/coin_balance.ex index e9930d0e0e..f8fadfcb53 100644 --- a/apps/indexer/lib/indexer/fetcher/coin_balance.ex +++ b/apps/indexer/lib/indexer/fetcher/coin_balance.ex @@ -14,6 +14,7 @@ defmodule Indexer.Fetcher.CoinBalance do alias EthereumJSONRPC.FetchedBalances alias Explorer.Chain alias Explorer.Chain.{Block, Hash} + alias Explorer.Chain.Cache.Accounts alias Indexer.{BufferedTask, Tracer} @behaviour BufferedTask @@ -136,7 +137,9 @@ defmodule Indexer.Fetcher.CoinBalance do end defp run_fetched_balances(%FetchedBalances{errors: errors} = fetched_balances, _) do - {:ok, _} = import_fetched_balances(fetched_balances) + {:ok, imported} = import_fetched_balances(fetched_balances) + + Accounts.drop(imported[:addresses]) retry(errors) end diff --git a/apps/indexer/lib/indexer/fetcher/coin_balance_on_demand.ex b/apps/indexer/lib/indexer/fetcher/coin_balance_on_demand.ex index d6d7d66d71..17014d2ede 100644 --- a/apps/indexer/lib/indexer/fetcher/coin_balance_on_demand.ex +++ b/apps/indexer/lib/indexer/fetcher/coin_balance_on_demand.ex @@ -19,7 +19,7 @@ defmodule Indexer.Fetcher.CoinBalanceOnDemand do alias Explorer.{Chain, Repo} alias Explorer.Chain.Address alias Explorer.Chain.Address.CoinBalance - alias Explorer.Chain.Cache.BlockNumber + alias Explorer.Chain.Cache.{Accounts, BlockNumber} alias Explorer.Counters.AverageBlockTime alias Indexer.Fetcher.CoinBalance, as: CoinBalanceFetcher alias Timex.Duration @@ -71,7 +71,11 @@ defmodule Indexer.Fetcher.CoinBalanceOnDemand do end def handle_cast({:fetch_and_update, block_number, address}, state) do - fetch_and_update(block_number, address, state.json_rpc_named_arguments) + result = fetch_and_update(block_number, address, state.json_rpc_named_arguments) + + with {:ok, %{addresses: addresses}} <- result do + Accounts.drop(addresses) + end {:noreply, state} end diff --git a/apps/indexer/lib/indexer/fetcher/contract_code.ex b/apps/indexer/lib/indexer/fetcher/contract_code.ex index bc95dd0d8e..cd9a5baa08 100644 --- a/apps/indexer/lib/indexer/fetcher/contract_code.ex +++ b/apps/indexer/lib/indexer/fetcher/contract_code.ex @@ -12,6 +12,7 @@ defmodule Indexer.Fetcher.ContractCode do alias Explorer.Chain alias Explorer.Chain.{Block, Hash} + alias Explorer.Chain.Cache.Accounts alias Indexer.{BufferedTask, Tracer} alias Indexer.Transform.Addresses @@ -126,7 +127,8 @@ defmodule Indexer.Fetcher.ContractCode do addresses: %{params: merged_addresses_params}, timeout: :infinity }) do - {:ok, _} -> + {:ok, imported} -> + Accounts.drop(imported[:addresses]) :ok {:error, step, reason, _changes_so_far} -> diff --git a/apps/indexer/lib/indexer/fetcher/internal_transaction.ex b/apps/indexer/lib/indexer/fetcher/internal_transaction.ex index 7d93bbf64f..5ad7c5d506 100644 --- a/apps/indexer/lib/indexer/fetcher/internal_transaction.ex +++ b/apps/indexer/lib/indexer/fetcher/internal_transaction.ex @@ -14,6 +14,7 @@ defmodule Indexer.Fetcher.InternalTransaction do alias Explorer.Chain alias Explorer.Chain.{Block, Hash} + alias Explorer.Chain.Cache.Accounts alias Indexer.{BufferedTask, Tracer} alias Indexer.Transform.Addresses @@ -218,6 +219,8 @@ defmodule Indexer.Fetcher.InternalTransaction do case imports do {:ok, imported} -> + Accounts.drop(imported[:addreses]) + async_import_coin_balances(imported, %{ address_hash_to_fetched_balance_block_number: address_hash_to_block_number }) diff --git a/apps/indexer/lib/indexer/fetcher/pending_transaction.ex b/apps/indexer/lib/indexer/fetcher/pending_transaction.ex index 7576d384d8..176d8dc9b7 100644 --- a/apps/indexer/lib/indexer/fetcher/pending_transaction.ex +++ b/apps/indexer/lib/indexer/fetcher/pending_transaction.ex @@ -14,6 +14,7 @@ defmodule Indexer.Fetcher.PendingTransaction do alias Ecto.Changeset alias Explorer.Chain + alias Explorer.Chain.Cache.Accounts alias Indexer.Fetcher.PendingTransaction alias Indexer.Transform.Addresses @@ -148,7 +149,8 @@ defmodule Indexer.Fetcher.PendingTransaction do broadcast: :realtime, transactions: %{params: transactions_params, on_conflict: :nothing} }) do - {:ok, _} -> + {:ok, imported} -> + Accounts.drop(imported[:addresses]) :ok {:error, [%Changeset{} | _] = changesets} -> diff --git a/apps/indexer/lib/indexer/fetcher/uncle_block.ex b/apps/indexer/lib/indexer/fetcher/uncle_block.ex index e9300d1e39..e597d16ca2 100644 --- a/apps/indexer/lib/indexer/fetcher/uncle_block.ex +++ b/apps/indexer/lib/indexer/fetcher/uncle_block.ex @@ -12,6 +12,7 @@ defmodule Indexer.Fetcher.UncleBlock do alias Ecto.Changeset alias EthereumJSONRPC.Blocks alias Explorer.Chain + alias Explorer.Chain.Cache.Accounts alias Explorer.Chain.Hash alias Indexer.{Block, BufferedTask, Tracer} alias Indexer.Fetcher.UncleBlock @@ -126,7 +127,8 @@ defmodule Indexer.Fetcher.UncleBlock do block_second_degree_relations: %{params: block_second_degree_relations_params}, transactions: %{params: transactions_params, on_conflict: :nothing} }) do - {:ok, _} -> + {:ok, imported} -> + Accounts.drop(imported[:addresses]) retry(errors) {:error, {:import = step, [%Changeset{} | _] = changesets}} -> From 432e6419dfcac6d7a1a400608cfa8f671824f708 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Wed, 18 Sep 2019 17:36:27 +0300 Subject: [PATCH 8/8] add comment --- apps/explorer/lib/explorer/chain.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index 1951f500a2..ce6995fc01 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -3074,6 +3074,7 @@ defmodule Explorer.Chain do |> normalize_balances_by_day() end + # https://github.com/poanetwork/blockscout/issues/2658 defp replace_last_value(items, %{value: value, timestamp: timestamp}) do List.replace_at(items, -1, %{date: Date.convert!(timestamp, Calendar.ISO), value: value}) end