Merge branch 'master' into ab-fix-total-supply

pull/2706/head
Victor Baranov 5 years ago committed by GitHub
commit 5f00209f71
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 3
      CHANGELOG.md
  2. 4
      apps/block_scout_web/assets/css/components/_dashboard-banner.scss
  3. 4
      apps/block_scout_web/test/block_scout_web/controllers/address_coin_balance_by_day_controller_test.exs
  4. 2
      apps/block_scout_web/test/support/conn_case.ex
  5. 2
      apps/block_scout_web/test/support/feature_case.ex
  6. 4
      apps/explorer/config/config.exs
  7. 4
      apps/explorer/lib/explorer/application.ex
  8. 39
      apps/explorer/lib/explorer/chain.ex
  9. 2
      apps/explorer/lib/explorer/chain/address/coin_balance.ex
  10. 73
      apps/explorer/lib/explorer/chain/cache/accounts.ex
  11. 42
      apps/explorer/test/explorer/chain/cache/accounts_test.exs
  12. 20
      apps/explorer/test/explorer/chain_test.exs
  13. 2
      apps/explorer/test/support/data_case.ex
  14. 5
      apps/indexer/lib/indexer/block/fetcher.ex
  15. 3
      apps/indexer/lib/indexer/block/realtime/fetcher.ex
  16. 5
      apps/indexer/lib/indexer/fetcher/block_reward.ex
  17. 5
      apps/indexer/lib/indexer/fetcher/coin_balance.ex
  18. 8
      apps/indexer/lib/indexer/fetcher/coin_balance_on_demand.ex
  19. 4
      apps/indexer/lib/indexer/fetcher/contract_code.ex
  20. 3
      apps/indexer/lib/indexer/fetcher/internal_transaction.ex
  21. 4
      apps/indexer/lib/indexer/fetcher/pending_transaction.ex
  22. 4
      apps/indexer/lib/indexer/fetcher/uncle_block.ex

@ -4,11 +4,13 @@
- [#2679](https://github.com/poanetwork/blockscout/pull/2679) - added fixed height for card chain blocks and card chain transactions
- [#2678](https://github.com/poanetwork/blockscout/pull/2678) - fixed dashboard banner height bug
- [#2672](https://github.com/poanetwork/blockscout/pull/2672) - added new theme for xUSDT
- [#2667](https://github.com/poanetwork/blockscout/pull/2667) - Add ETS-based cache for accounts page
- [#2666](https://github.com/poanetwork/blockscout/pull/2666) - fetch token counters in parallel
- [#2665](https://github.com/poanetwork/blockscout/pull/2665) - new menu layout for mobile devices
- [#2663](https://github.com/poanetwork/blockscout/pull/2663) - Fetch address counters in parallel
### Fixes
- [#2707](https://github.com/poanetwork/blockscout/pull/2707) - fix for dashboard banner chart legend items
- [#2706](https://github.com/poanetwork/blockscout/pull/2706) - fix empty total_supply in coin gecko response
- [#2701](https://github.com/poanetwork/blockscout/pull/2701) - Exclude nonconsensus blocks from avg block time calculation by default
- [#2696](https://github.com/poanetwork/blockscout/pull/2696) - do not update fetched_coin_balance with nil
@ -18,6 +20,7 @@
- [#2684](https://github.com/poanetwork/blockscout/pull/2684) - do not filter pending logs
- [#2682](https://github.com/poanetwork/blockscout/pull/2682) - Use Task.start instead of Task.async in caches
- [#2671](https://github.com/poanetwork/blockscout/pull/2671) - fixed buttons color at smart contract section
- [#2660](https://github.com/poanetwork/blockscout/pull/2660) - set correct last value for coin balances chart data
- [#2619](https://github.com/poanetwork/blockscout/pull/2619) - Enforce DB transaction's order to prevent deadlocks
### Chore

@ -72,7 +72,7 @@ $dashboard-banner-chart-axis-font-color: $dashboard-stats-item-value-color !defa
}
.dashboard-banner-chart-legend {
display: flex;
display: grid;
grid-template-columns: 1fr 1fr;
padding-bottom: 12px;
@ -81,7 +81,7 @@ $dashboard-banner-chart-axis-font-color: $dashboard-stats-item-value-color !defa
padding-left: 12px;
padding-top: 3px;
position: relative;
padding-right: 60px;
padding-right: 12px;
@include media-breakpoint-down(md) {
display: flex;

@ -5,8 +5,8 @@ defmodule BlockScoutWeb.AddressCoinBalanceByDayControllerTest do
test "returns the coin balance history grouped by date", %{conn: conn} do
address = insert(:address)
noon = Timex.now() |> Timex.beginning_of_day() |> Timex.set(hour: 12)
block = insert(:block, timestamp: noon)
block_one_day_ago = insert(:block, timestamp: Timex.shift(noon, days: -1))
block = insert(:block, timestamp: noon, number: 2)
block_one_day_ago = insert(:block, timestamp: Timex.shift(noon, days: -1), number: 1)
insert(:fetched_balance, address_hash: address.hash, value: 1000, block_number: block.number)
insert(:fetched_balance, address_hash: address.hash, value: 2000, block_number: block_one_day_ago.number)

@ -42,6 +42,8 @@ defmodule BlockScoutWeb.ConnCase do
Supervisor.terminate_child(Explorer.Supervisor, Explorer.Chain.Cache.Transactions.child_id())
Supervisor.restart_child(Explorer.Supervisor, Explorer.Chain.Cache.Transactions.child_id())
Supervisor.terminate_child(Explorer.Supervisor, Explorer.Chain.Cache.Accounts.child_id())
Supervisor.restart_child(Explorer.Supervisor, Explorer.Chain.Cache.Accounts.child_id())
{:ok, conn: Phoenix.ConnTest.build_conn()}
end

@ -29,6 +29,8 @@ defmodule BlockScoutWeb.FeatureCase do
Supervisor.terminate_child(Explorer.Supervisor, Explorer.Chain.Cache.Transactions.child_id())
Supervisor.restart_child(Explorer.Supervisor, Explorer.Chain.Cache.Transactions.child_id())
Supervisor.terminate_child(Explorer.Supervisor, Explorer.Chain.Cache.Accounts.child_id())
Supervisor.restart_child(Explorer.Supervisor, Explorer.Chain.Cache.Accounts.child_id())
metadata = Phoenix.Ecto.SQL.Sandbox.metadata_for(Explorer.Repo, self())
{:ok, session} = Wallaby.start_session(metadata: metadata)

@ -136,6 +136,10 @@ config :explorer, Explorer.Chain.Cache.Transactions,
ttl_check_interval: if(System.get_env("DISABLE_INDEXER") == "true", do: :timer.seconds(1), else: false),
global_ttl: if(System.get_env("DISABLE_INDEXER") == "true", do: :timer.seconds(5))
config :explorer, Explorer.Chain.Cache.Accounts,
ttl_check_interval: if(System.get_env("DISABLE_INDEXER") == "true", do: :timer.seconds(1), else: false),
global_ttl: if(System.get_env("DISABLE_INDEXER") == "true", do: :timer.seconds(5))
# Import environment specific config. This must remain at the bottom
# of this file so it overrides the configuration defined above.
import_config "#{Mix.env()}.exs"

@ -8,6 +8,7 @@ defmodule Explorer.Application do
alias Explorer.Admin
alias Explorer.Chain.Cache.{
Accounts,
BlockCount,
BlockNumber,
Blocks,
@ -49,7 +50,8 @@ defmodule Explorer.Application do
BlockNumber,
con_cache_child_spec(MarketHistoryCache.cache_name()),
con_cache_child_spec(RSK.cache_name(), ttl_check_interval: :timer.minutes(1), global_ttl: :timer.minutes(30)),
Transactions
Transactions,
Accounts
]
children = base_children ++ configurable_children()

@ -49,6 +49,7 @@ defmodule Explorer.Chain do
alias Explorer.Chain.Block.{EmissionReward, Reward}
alias Explorer.Chain.Cache.{
Accounts,
BlockCount,
BlockNumber,
Blocks,
@ -1379,6 +1380,36 @@ defmodule Explorer.Chain do
def list_top_addresses(options \\ []) do
paging_options = Keyword.get(options, :paging_options, @default_paging_options)
if is_nil(paging_options.key) do
paging_options.page_size
|> Accounts.take_enough()
|> case do
nil ->
accounts_with_n = fetch_top_addresses(paging_options)
accounts_with_n
|> Enum.map(fn {address, _n} -> address end)
|> Accounts.update()
accounts_with_n
accounts ->
Enum.map(
accounts,
&{&1,
if is_nil(&1.nonce) do
0
else
&1.nonce + 1
end}
)
end
else
fetch_top_addresses(paging_options)
end
end
defp fetch_top_addresses(paging_options) do
base_query =
from(a in Address,
where: a.fetched_coin_balance > ^0,
@ -3097,9 +3128,17 @@ defmodule Explorer.Chain do
address_hash
|> CoinBalance.balances_by_day(latest_block_timestamp)
|> Repo.all()
|> replace_last_value(latest_block_timestamp)
|> normalize_balances_by_day()
end
# https://github.com/poanetwork/blockscout/issues/2658
defp replace_last_value(items, %{value: value, timestamp: timestamp}) do
List.replace_at(items, -1, %{date: Date.convert!(timestamp, Calendar.ISO), value: value})
end
defp replace_last_value(items, _), do: items
defp normalize_balances_by_day(balances_by_day) do
result =
balances_by_day

@ -112,7 +112,7 @@ defmodule Explorer.Chain.Address.CoinBalance do
|> join(:inner, [cb], b in Block, on: cb.block_number == b.number)
|> where([cb], cb.address_hash == ^address_hash)
|> last(:block_number)
|> select([cb, b], %{timestamp: b.timestamp})
|> select([cb, b], %{timestamp: b.timestamp, value: cb.value})
end
def changeset(%__MODULE__{} = balance, params) do

@ -0,0 +1,73 @@
defmodule Explorer.Chain.Cache.Accounts do
@moduledoc """
Caches the top Addresses
"""
alias Explorer.Chain.Address
use Explorer.Chain.OrderedCache,
name: :accounts,
max_size: 51,
preload: :names,
ttl_check_interval: Application.get_env(:explorer, __MODULE__)[:ttl_check_interval],
global_ttl: Application.get_env(:explorer, __MODULE__)[:global_ttl]
@type element :: Address.t()
@type id :: {non_neg_integer(), non_neg_integer()}
def element_to_id(%Address{fetched_coin_balance: fetched_coin_balance, hash: hash}) do
{fetched_coin_balance, hash}
end
def prevails?({fetched_coin_balance_a, hash_a}, {fetched_coin_balance_b, hash_b}) do
# same as a query's `order_by: [desc: :fetched_coin_balance, asc: :hash]`
if fetched_coin_balance_a == fetched_coin_balance_b do
hash_a < hash_b
else
fetched_coin_balance_a > fetched_coin_balance_b
end
end
def drop(nil), do: :ok
def drop([]), do: :ok
def drop(addresses) when is_list(addresses) do
# This has to be used by the Indexer insead of `update`.
# The reason being that addresses already in the cache can change their balance
# value and removing or updating them will result into a potentially invalid
# cache status, that would not even get corrected with time.
# The only thing we can safely do when an address in the cache changes its
# `fetched_coin_balance` is to invalidate the whole cache and wait for it
# to be filled again (by the query that it takes the place of when full).
ConCache.update(cache_name(), ids_list_key(), fn ids ->
if drop_needed?(ids, addresses) do
# Remove the addresses immediately
Enum.each(ids, &ConCache.delete(cache_name(), &1))
{:ok, []}
else
{:ok, ids}
end
end)
end
def drop(address), do: drop([address])
defp drop_needed?(ids, _addresses) when is_nil(ids), do: false
defp drop_needed?([], _addresses), do: false
defp drop_needed?(ids, addresses) do
ids_map = Map.new(ids, fn {balance, hash} -> {hash, balance} end)
# Result it `true` only when the address is present in the cache already,
# but with a different `fetched_coin_balance`
Enum.find_value(addresses, false, fn address ->
stored_address_balance = Map.get(ids_map, address.hash)
stored_address_balance && stored_address_balance != address.fetched_coin_balance
end)
end
end

@ -0,0 +1,42 @@
defmodule Explorer.Chain.Cache.AccountsTest do
use Explorer.DataCase
alias Explorer.Chain.Cache.Accounts
alias Explorer.Repo
describe "drop/1" do
test "does not drop the cache if the address fetched_coin_balance has not changed" do
address =
insert(:address, fetched_coin_balance: 100_000, fetched_coin_balance_block_number: 1)
|> preload_names()
Accounts.update(address)
assert Accounts.take(1) == [address]
Accounts.drop(address)
assert Accounts.take(1) == [address]
end
test "drops the cache if an address was in the cache with a different fetched_coin_balance" do
address =
insert(:address, fetched_coin_balance: 100_000, fetched_coin_balance_block_number: 1)
|> preload_names()
Accounts.update(address)
assert Accounts.take(1) == [address]
updated_address = %{address | fetched_coin_balance: 100_001}
Accounts.drop(updated_address)
assert Accounts.take(1) == []
end
end
defp preload_names(address) do
Repo.preload(address, [:names])
end
end

@ -3879,9 +3879,9 @@ defmodule Explorer.ChainTest do
address = insert(:address)
today = NaiveDateTime.utc_now()
noon = Timex.set(today, hour: 12)
block = insert(:block, timestamp: noon)
block = insert(:block, timestamp: noon, number: 50)
yesterday = Timex.shift(noon, days: -1)
block_one_day_ago = insert(:block, timestamp: yesterday)
block_one_day_ago = insert(:block, timestamp: yesterday, number: 49)
insert(:fetched_balance, address_hash: address.hash, value: 1000, block_number: block.number)
insert(:fetched_balance, address_hash: address.hash, value: 2000, block_number: block_one_day_ago.number)
@ -3908,6 +3908,22 @@ defmodule Explorer.ChainTest do
%{date: today |> NaiveDateTime.to_date() |> Date.to_string(), value: Decimal.new("1E-15")}
]
end
test "uses last block value if there a couple of change in the same day" do
address = insert(:address)
today = NaiveDateTime.utc_now()
past = Timex.shift(today, hours: -1)
block_now = insert(:block, timestamp: today, number: 1)
insert(:fetched_balance, address_hash: address.hash, value: 1, block_number: block_now.number)
block_past = insert(:block, timestamp: past, number: 2)
insert(:fetched_balance, address_hash: address.hash, value: 0, block_number: block_past.number)
[balance] = Chain.address_to_balances_by_day(address.hash)
assert balance.value == Decimal.new(0)
end
end
describe "block_combined_rewards/1" do

@ -45,6 +45,8 @@ defmodule Explorer.DataCase do
Supervisor.restart_child(Explorer.Supervisor, Explorer.Chain.Cache.Blocks.child_id())
Supervisor.terminate_child(Explorer.Supervisor, Explorer.Chain.Cache.Transactions.child_id())
Supervisor.restart_child(Explorer.Supervisor, Explorer.Chain.Cache.Transactions.child_id())
Supervisor.terminate_child(Explorer.Supervisor, Explorer.Chain.Cache.Accounts.child_id())
Supervisor.restart_child(Explorer.Supervisor, Explorer.Chain.Cache.Accounts.child_id())
:ok
end

@ -13,7 +13,7 @@ defmodule Indexer.Block.Fetcher do
alias Explorer.Chain
alias Explorer.Chain.{Address, Block, Hash, Import, Transaction}
alias Explorer.Chain.Cache.Blocks, as: BlocksCache
alias Explorer.Chain.Cache.{BlockNumber, Transactions}
alias Explorer.Chain.Cache.{Accounts, BlockNumber, Transactions}
alias Indexer.Block.Fetcher.Receipts
alias Indexer.Fetcher.{
@ -176,6 +176,7 @@ defmodule Indexer.Block.Fetcher do
result = {:ok, %{inserted: inserted, errors: blocks_errors}}
update_block_cache(inserted[:blocks])
update_transactions_cache(inserted[:transactions])
update_addresses_cache(inserted[:addresses])
result
else
{step, {:error, reason}} -> {:error, {step, reason}}
@ -197,6 +198,8 @@ defmodule Indexer.Block.Fetcher do
Transactions.update(transactions)
end
defp update_addresses_cache(addresses), do: Accounts.drop(addresses)
def import(
%__MODULE__{broadcast: broadcast, callback_module: callback_module} = state,
options

@ -27,6 +27,7 @@ defmodule Indexer.Block.Realtime.Fetcher do
alias Ecto.Changeset
alias EthereumJSONRPC.{FetchedBalances, Subscription}
alias Explorer.Chain
alias Explorer.Chain.Cache.Accounts
alias Explorer.Counters.AverageBlockTime
alias Indexer.{Block, Tracer}
alias Indexer.Block.Realtime.TaskSupervisor
@ -197,6 +198,8 @@ defmodule Indexer.Block.Realtime.Fetcher do
json_rpc_named_arguments
)
Accounts.drop(imported[:addresses])
ok
end
end

@ -17,6 +17,7 @@ defmodule Indexer.Fetcher.BlockReward do
alias EthereumJSONRPC.FetchedBeneficiaries
alias Explorer.Chain
alias Explorer.Chain.{Block, Wei}
alias Explorer.Chain.Cache.Accounts
alias Indexer.{BufferedTask, Tracer}
alias Indexer.Fetcher.BlockReward.Supervisor, as: BlockRewardSupervisor
alias Indexer.Fetcher.CoinBalance
@ -130,7 +131,9 @@ defmodule Indexer.Fetcher.BlockReward do
|> add_gas_payments()
|> import_block_reward_params()
|> case do
{:ok, %{address_coin_balances: address_coin_balances}} ->
{:ok, %{address_coin_balances: address_coin_balances, addresses: addresses}} ->
Accounts.drop(addresses)
CoinBalance.async_fetch_balances(address_coin_balances)
retry_errors(errors)

@ -14,6 +14,7 @@ defmodule Indexer.Fetcher.CoinBalance do
alias EthereumJSONRPC.FetchedBalances
alias Explorer.Chain
alias Explorer.Chain.{Block, Hash}
alias Explorer.Chain.Cache.Accounts
alias Indexer.{BufferedTask, Tracer}
@behaviour BufferedTask
@ -136,7 +137,9 @@ defmodule Indexer.Fetcher.CoinBalance do
end
defp run_fetched_balances(%FetchedBalances{errors: errors} = fetched_balances, _) do
{:ok, _} = import_fetched_balances(fetched_balances)
{:ok, imported} = import_fetched_balances(fetched_balances)
Accounts.drop(imported[:addresses])
retry(errors)
end

@ -19,7 +19,7 @@ defmodule Indexer.Fetcher.CoinBalanceOnDemand do
alias Explorer.{Chain, Repo}
alias Explorer.Chain.Address
alias Explorer.Chain.Address.CoinBalance
alias Explorer.Chain.Cache.BlockNumber
alias Explorer.Chain.Cache.{Accounts, BlockNumber}
alias Explorer.Counters.AverageBlockTime
alias Indexer.Fetcher.CoinBalance, as: CoinBalanceFetcher
alias Timex.Duration
@ -71,7 +71,11 @@ defmodule Indexer.Fetcher.CoinBalanceOnDemand do
end
def handle_cast({:fetch_and_update, block_number, address}, state) do
fetch_and_update(block_number, address, state.json_rpc_named_arguments)
result = fetch_and_update(block_number, address, state.json_rpc_named_arguments)
with {:ok, %{addresses: addresses}} <- result do
Accounts.drop(addresses)
end
{:noreply, state}
end

@ -12,6 +12,7 @@ defmodule Indexer.Fetcher.ContractCode do
alias Explorer.Chain
alias Explorer.Chain.{Block, Hash}
alias Explorer.Chain.Cache.Accounts
alias Indexer.{BufferedTask, Tracer}
alias Indexer.Transform.Addresses
@ -126,7 +127,8 @@ defmodule Indexer.Fetcher.ContractCode do
addresses: %{params: merged_addresses_params},
timeout: :infinity
}) do
{:ok, _} ->
{:ok, imported} ->
Accounts.drop(imported[:addresses])
:ok
{:error, step, reason, _changes_so_far} ->

@ -14,6 +14,7 @@ defmodule Indexer.Fetcher.InternalTransaction do
alias Explorer.Chain
alias Explorer.Chain.{Block, Hash}
alias Explorer.Chain.Cache.Accounts
alias Indexer.{BufferedTask, Tracer}
alias Indexer.Transform.Addresses
@ -218,6 +219,8 @@ defmodule Indexer.Fetcher.InternalTransaction do
case imports do
{:ok, imported} ->
Accounts.drop(imported[:addreses])
async_import_coin_balances(imported, %{
address_hash_to_fetched_balance_block_number: address_hash_to_block_number
})

@ -14,6 +14,7 @@ defmodule Indexer.Fetcher.PendingTransaction do
alias Ecto.Changeset
alias Explorer.Chain
alias Explorer.Chain.Cache.Accounts
alias Indexer.Fetcher.PendingTransaction
alias Indexer.Transform.Addresses
@ -148,7 +149,8 @@ defmodule Indexer.Fetcher.PendingTransaction do
broadcast: :realtime,
transactions: %{params: transactions_params, on_conflict: :nothing}
}) do
{:ok, _} ->
{:ok, imported} ->
Accounts.drop(imported[:addresses])
:ok
{:error, [%Changeset{} | _] = changesets} ->

@ -12,6 +12,7 @@ defmodule Indexer.Fetcher.UncleBlock do
alias Ecto.Changeset
alias EthereumJSONRPC.Blocks
alias Explorer.Chain
alias Explorer.Chain.Cache.Accounts
alias Explorer.Chain.Hash
alias Indexer.{Block, BufferedTask, Tracer}
alias Indexer.Fetcher.UncleBlock
@ -126,7 +127,8 @@ defmodule Indexer.Fetcher.UncleBlock do
block_second_degree_relations: %{params: block_second_degree_relations_params},
transactions: %{params: transactions_params, on_conflict: :nothing}
}) do
{:ok, _} ->
{:ok, imported} ->
Accounts.drop(imported[:addresses])
retry(errors)
{:error, {:import = step, [%Changeset{} | _] = changesets}} ->

Loading…
Cancel
Save