From 8b3d032c16066977823e1906817bc703227ebdd7 Mon Sep 17 00:00:00 2001 From: Yegor San Date: Fri, 30 Aug 2019 21:43:10 +0300 Subject: [PATCH 01/67] Fixed buttons color at smart contract section --- .../templates/smart_contract/_functions.html.eex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/block_scout_web/lib/block_scout_web/templates/smart_contract/_functions.html.eex b/apps/block_scout_web/lib/block_scout_web/templates/smart_contract/_functions.html.eex index 3008b37cf8..894daecbb8 100644 --- a/apps/block_scout_web/lib/block_scout_web/templates/smart_contract/_functions.html.eex +++ b/apps/block_scout_web/lib/block_scout_web/templates/smart_contract/_functions.html.eex @@ -19,7 +19,7 @@ <% end %> - +
From 326b70fa5289fbdab484049b12ac33407d52e931 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 3 Sep 2019 11:48:28 +0300 Subject: [PATCH 02/67] set correct last calue for coin balances chart data Sometimes the last value for coin history chart data is incorrect because when selecting records they are grouped by day and max value is selected in that day. So if there are a couple of coin balance changes only max value is shown. This PR always shows the latest value from the database by block number. --- apps/explorer/lib/explorer/chain.ex | 7 +++++++ .../lib/explorer/chain/address/coin_balance.ex | 2 +- apps/explorer/test/explorer/chain_test.exs | 16 ++++++++++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index 1581bd4846..b2a7c6fdb1 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -3002,9 +3002,16 @@ defmodule Explorer.Chain do address_hash |> CoinBalance.balances_by_day(latest_block_timestamp) |> Repo.all() + |> replace_last_value(latest_block_timestamp) |> normalize_balances_by_day() end + defp replace_last_value(items, %{value: value, timestamp: timestamp}) do + List.replace_at(items, -1, %{date: Date.convert!(timestamp, Calendar.ISO), value: value}) + end + + defp replace_last_value(items, _), do: items + defp normalize_balances_by_day(balances_by_day) do result = balances_by_day diff --git a/apps/explorer/lib/explorer/chain/address/coin_balance.ex b/apps/explorer/lib/explorer/chain/address/coin_balance.ex index 537f6a0483..f3647a8f54 100644 --- a/apps/explorer/lib/explorer/chain/address/coin_balance.ex +++ b/apps/explorer/lib/explorer/chain/address/coin_balance.ex @@ -112,7 +112,7 @@ defmodule Explorer.Chain.Address.CoinBalance do |> join(:inner, [cb], b in Block, on: cb.block_number == b.number) |> where([cb], cb.address_hash == ^address_hash) |> last(:block_number) - |> select([cb, b], %{timestamp: b.timestamp}) + |> select([cb, b], %{timestamp: b.timestamp, value: cb.value}) end def changeset(%__MODULE__{} = balance, params) do diff --git a/apps/explorer/test/explorer/chain_test.exs b/apps/explorer/test/explorer/chain_test.exs index 187d503621..d9bfec58f4 100644 --- a/apps/explorer/test/explorer/chain_test.exs +++ b/apps/explorer/test/explorer/chain_test.exs @@ -3908,6 +3908,22 @@ defmodule Explorer.ChainTest do %{date: today |> NaiveDateTime.to_date() |> Date.to_string(), value: Decimal.new("1E-15")} ] end + + test "uses last block value if there a couple of change in the same day" do + address = insert(:address) + today = NaiveDateTime.utc_now() + past = Timex.shift(today, hours: -1) + + block_now = insert(:block, timestamp: today) + insert(:fetched_balance, address_hash: address.hash, value: 1, block_number: block_now.number) + + block_past = insert(:block, timestamp: past) + insert(:fetched_balance, address_hash: address.hash, value: 0, block_number: block_past.number) + + [balance] = Chain.address_to_balances_by_day(address.hash) + + assert balance.value == Decimal.new(0) + end end describe "block_combined_rewards/1" do From bf23cc0ec17d0e99bfbccd9c06cc3450a196a9f0 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 3 Sep 2019 11:54:28 +0300 Subject: [PATCH 03/67] add CHANGELOG entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca60257372..8ccd24aa31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - [#2497](https://github.com/poanetwork/blockscout/pull/2497) - Add generic Ordered Cache behaviour and implementation ### Fixes +- [#2660](https://github.com/poanetwork/blockscout/pull/2660) - set correct last value for coin balances chart data - [#2468](https://github.com/poanetwork/blockscout/pull/2468) - fix confirmations for non consensus blocks - [#2610](https://github.com/poanetwork/blockscout/pull/2610) - use CoinGecko instead of CoinMarketcap for exchange rates - [#2640](https://github.com/poanetwork/blockscout/pull/2640) - SVG network icons From acd7fb53b31200f6995a2bfad67f3c3a9286c05e Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 3 Sep 2019 11:58:11 +0300 Subject: [PATCH 04/67] fix test --- apps/explorer/test/explorer/chain_test.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/explorer/test/explorer/chain_test.exs b/apps/explorer/test/explorer/chain_test.exs index d9bfec58f4..d673336ef6 100644 --- a/apps/explorer/test/explorer/chain_test.exs +++ b/apps/explorer/test/explorer/chain_test.exs @@ -3914,10 +3914,10 @@ defmodule Explorer.ChainTest do today = NaiveDateTime.utc_now() past = Timex.shift(today, hours: -1) - block_now = insert(:block, timestamp: today) + block_now = insert(:block, timestamp: today, number: 1) insert(:fetched_balance, address_hash: address.hash, value: 1, block_number: block_now.number) - block_past = insert(:block, timestamp: past) + block_past = insert(:block, timestamp: past, number: 2) insert(:fetched_balance, address_hash: address.hash, value: 0, block_number: block_past.number) [balance] = Chain.address_to_balances_by_day(address.hash) From 05a00b4cc90eb981e2dd118e22abc5635e3ce8ce Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 3 Sep 2019 12:25:57 +0300 Subject: [PATCH 05/67] fix tests --- .../address_coin_balance_by_day_controller_test.exs | 4 ++-- apps/explorer/test/explorer/chain_test.exs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/block_scout_web/test/block_scout_web/controllers/address_coin_balance_by_day_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/address_coin_balance_by_day_controller_test.exs index 1496bb24e9..df714247c2 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/address_coin_balance_by_day_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/address_coin_balance_by_day_controller_test.exs @@ -5,8 +5,8 @@ defmodule BlockScoutWeb.AddressCoinBalanceByDayControllerTest do test "returns the coin balance history grouped by date", %{conn: conn} do address = insert(:address) noon = Timex.now() |> Timex.beginning_of_day() |> Timex.set(hour: 12) - block = insert(:block, timestamp: noon) - block_one_day_ago = insert(:block, timestamp: Timex.shift(noon, days: -1)) + block = insert(:block, timestamp: noon, number: 2) + block_one_day_ago = insert(:block, timestamp: Timex.shift(noon, days: -1), number: 1) insert(:fetched_balance, address_hash: address.hash, value: 1000, block_number: block.number) insert(:fetched_balance, address_hash: address.hash, value: 2000, block_number: block_one_day_ago.number) diff --git a/apps/explorer/test/explorer/chain_test.exs b/apps/explorer/test/explorer/chain_test.exs index d673336ef6..3ac60b6963 100644 --- a/apps/explorer/test/explorer/chain_test.exs +++ b/apps/explorer/test/explorer/chain_test.exs @@ -3879,9 +3879,9 @@ defmodule Explorer.ChainTest do address = insert(:address) today = NaiveDateTime.utc_now() noon = Timex.set(today, hour: 12) - block = insert(:block, timestamp: noon) + block = insert(:block, timestamp: noon, number: 50) yesterday = Timex.shift(noon, days: -1) - block_one_day_ago = insert(:block, timestamp: yesterday) + block_one_day_ago = insert(:block, timestamp: yesterday, number: 49) insert(:fetched_balance, address_hash: address.hash, value: 1000, block_number: block.number) insert(:fetched_balance, address_hash: address.hash, value: 2000, block_number: block_one_day_ago.number) From decfc08a54e3b11057450b0251d9a8c2ea960936 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 3 Sep 2019 12:46:33 +0300 Subject: [PATCH 06/67] fetch coin gecko id based on coin symbol Coin Gecko provides API with all listed coins. We can use this API to find coin id instead of setting it by hand. --- .../exchange_rates/source/coin_gecko.ex | 32 ++++++++++++++-- .../exchange_rates/source/coin_gecko_test.exs | 37 +++++++++++++++++++ docs/env-variables.md | 1 - 3 files changed, 66 insertions(+), 4 deletions(-) diff --git a/apps/explorer/lib/explorer/exchange_rates/source/coin_gecko.ex b/apps/explorer/lib/explorer/exchange_rates/source/coin_gecko.ex index 95ecafb046..5a6199acb2 100644 --- a/apps/explorer/lib/explorer/exchange_rates/source/coin_gecko.ex +++ b/apps/explorer/lib/explorer/exchange_rates/source/coin_gecko.ex @@ -44,15 +44,41 @@ defmodule Explorer.ExchangeRates.Source.CoinGecko do @impl Source def source_url do - "#{base_url()}/coins/#{coin_id()}" + {:ok, id} = coin_id() + + "#{base_url()}/coins/#{id}" end defp base_url do config(:base_url) || "https://api.coingecko.com/api/v3" end - defp coin_id do - Application.get_env(:explorer, __MODULE__)[:coin_id] + def coin_id do + url = "#{base_url()}/coins/list" + + symbol = String.downcase(Explorer.coin()) + + case HTTPoison.get(url, headers()) do + {:ok, %Response{body: body, status_code: 200}} -> + data = decode_json(body) + + symbol_data = + Enum.find(data, fn item -> + item["symbol"] == symbol + end) + + if symbol_data do + {:ok, symbol_data["id"]} + else + {:errpr, :not_found} + end + + {:ok, %Response{body: body, status_code: status_code}} when status_code in 400..499 -> + {:error, decode_json(body)["error"]} + + {:error, %Error{reason: reason}} -> + {:error, reason} + end end defp get_btc_price(currency \\ "usd") do diff --git a/apps/explorer/test/explorer/exchange_rates/source/coin_gecko_test.exs b/apps/explorer/test/explorer/exchange_rates/source/coin_gecko_test.exs index b4b3313d6c..3072099cbc 100644 --- a/apps/explorer/test/explorer/exchange_rates/source/coin_gecko_test.exs +++ b/apps/explorer/test/explorer/exchange_rates/source/coin_gecko_test.exs @@ -18,6 +18,26 @@ defmodule Explorer.ExchangeRates.Source.CoinGeckoTest do } """ + @coins_list """ + [ + { + "id": "poa-network", + "symbol": "poa", + "name": "POA Network" + }, + { + "id": "poc-chain", + "symbol": "pocc", + "name": "POC Chain" + }, + { + "id": "pocket-arena", + "symbol": "poc", + "name": "Pocket Arena" + } + ] + """ + describe "format_data/1" do setup do bypass = Bypass.open() @@ -62,4 +82,21 @@ defmodule Explorer.ExchangeRates.Source.CoinGeckoTest do assert [] = CoinGecko.format_data(bad_data) end end + + describe "coin_id/0" do + setup do + bypass = Bypass.open() + Application.put_env(:explorer, CoinGecko, base_url: "http://localhost:#{bypass.port}") + + {:ok, bypass: bypass} + end + + test "fetches coin id", %{bypass: bypass} do + Bypass.expect(bypass, "GET", "/coins/list", fn conn -> + Conn.resp(conn, 200, @coins_list) + end) + + assert CoinGecko.coin_id() == {:ok, "poa-network"} + end + end end diff --git a/docs/env-variables.md b/docs/env-variables.md index dd01cbe26f..39f90f6336 100644 --- a/docs/env-variables.md +++ b/docs/env-variables.md @@ -65,4 +65,3 @@ $ export NETWORK=POA | `WEBAPP_URL` | | Link to web application instance, e.g. `http://host/path` | (empty) | v2.0.3+ | | | | `API_URL` | | Link to API instance, e.g. `http://host/path` | (empty) | v2.0.3+ | | | | `CHAIN_SPEC_PATH` | | Chain specification path (absolute file system path or url) to import block emission reward ranges and genesis account balances from | (empty) | master | | | -| `COIN_GECKO_ID` | | CoinGecko coin id required for fetching an exchange rate | poa-network | master | | | From 39c01bc8a2b44b2fe5bedd9d6f43ad131ca8d9b3 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 3 Sep 2019 12:49:13 +0300 Subject: [PATCH 07/67] add CHANGELOG entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca60257372..7000f890fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ - [#2538](https://github.com/poanetwork/blockscout/pull/2538) - fetch the last not empty coin balance records ### Chore +- [#2662](https://github.com/poanetwork/blockscout/pull/2662) - fetch coin gecko id based on the coin symbol - [#2646](https://github.com/poanetwork/blockscout/pull/2646) - Added Xerom to list of Additional Chains using BlockScout - [#2634](https://github.com/poanetwork/blockscout/pull/2634) - add Lukso to networks dropdown - [#2617](https://github.com/poanetwork/blockscout/pull/2617) - skip cache update if there are no blocks inserted From 213395ec32cd1d8201bd849aa94399f1cc425b7f Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 3 Sep 2019 12:50:29 +0300 Subject: [PATCH 08/67] remove config entry --- apps/explorer/config/config.exs | 2 -- 1 file changed, 2 deletions(-) diff --git a/apps/explorer/config/config.exs b/apps/explorer/config/config.exs index 10e8a66e82..9923625359 100644 --- a/apps/explorer/config/config.exs +++ b/apps/explorer/config/config.exs @@ -31,8 +31,6 @@ config :explorer, Explorer.ChainSpec.GenesisData, enabled: false, chain_spec_pat config :explorer, Explorer.Chain.Cache.BlockNumber, enabled: true -config :explorer, Explorer.ExchangeRates.Source.CoinGecko, coin_id: System.get_env("COIN_GECKO_ID", "poa-network") - balances_update_interval = if System.get_env("ADDRESS_WITH_BALANCES_UPDATE_INTERVAL") do case Integer.parse(System.get_env("ADDRESS_WITH_BALANCES_UPDATE_INTERVAL")) do From 36c1aa9b67ddf0424142623ce8cab7db5f2a2716 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Wed, 4 Sep 2019 15:52:25 +0300 Subject: [PATCH 09/67] fetch token counters in parallel the slowest part of the token overview page is fetching of total token transfers and total token holders counters queries. Sometimes they even cause page load failures. Now we will start fetching these counters in parallel and limiting query execution to 40 seconds. If they fail to load, we won't show these counters. --- .../controllers/tokens/holder_controller.ex | 8 +++-- .../tokens/inventory_controller.ex | 7 ++-- .../tokens/read_contract_controller.ex | 8 +++-- .../controllers/tokens/token_controller.ex | 34 +++++++++++++++++++ .../controllers/tokens/transfer_controller.ex | 7 ++-- .../tokens/overview/_details.html.eex | 8 +++-- 6 files changed, 62 insertions(+), 10 deletions(-) diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/tokens/holder_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/tokens/holder_controller.ex index 4de35fc012..553efef03e 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/tokens/holder_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/tokens/holder_controller.ex @@ -12,6 +12,8 @@ defmodule BlockScoutWeb.Tokens.HolderController do next_page_params: 3 ] + import BlockScoutWeb.Tokens.TokenController, only: [fetch_token_counters: 2] + def index(conn, %{"token_id" => address_hash_string, "type" => "JSON"} = params) do with {:ok, address_hash} <- Chain.string_to_address_hash(address_hash_string), {:ok, token} <- Chain.token_from_address_hash(address_hash), @@ -47,13 +49,15 @@ defmodule BlockScoutWeb.Tokens.HolderController do with {:ok, address_hash} <- Chain.string_to_address_hash(address_hash_string), {:ok, token} <- Chain.token_from_address_hash(address_hash, options) do + {total_token_transfers, total_token_holders} = fetch_token_counters(token, address_hash) + render( conn, "index.html", current_path: current_path(conn), token: Market.add_price(token), - total_token_holders: token.holder_count || Chain.count_token_holders_from_token_hash(address_hash), - total_token_transfers: Chain.count_token_transfers_from_token_hash(address_hash) + total_token_holders: total_token_holders, + total_token_transfers: total_token_transfers ) else :error -> diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/tokens/inventory_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/tokens/inventory_controller.ex index b9783b2c63..ddd12eb392 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/tokens/inventory_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/tokens/inventory_controller.ex @@ -7,6 +7,7 @@ defmodule BlockScoutWeb.Tokens.InventoryController do alias Phoenix.View import BlockScoutWeb.Chain, only: [split_list_by_page: 1, default_paging_options: 0] + import BlockScoutWeb.Tokens.TokenController, only: [fetch_token_counters: 2] def index(conn, %{"token_id" => address_hash_string, "type" => "JSON"} = params) do with {:ok, address_hash} <- Chain.string_to_address_hash(address_hash_string), @@ -64,13 +65,15 @@ defmodule BlockScoutWeb.Tokens.InventoryController do with {:ok, address_hash} <- Chain.string_to_address_hash(address_hash_string), {:ok, token} <- Chain.token_from_address_hash(address_hash, options) do + {total_token_transfers, total_token_holders} = fetch_token_counters(token, address_hash) + render( conn, "index.html", current_path: current_path(conn), token: Market.add_price(token), - total_token_transfers: Chain.count_token_transfers_from_token_hash(address_hash), - total_token_holders: token.holder_count || Chain.count_token_holders_from_token_hash(address_hash) + total_token_transfers: total_token_transfers, + total_token_holders: total_token_holders ) else :error -> diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/tokens/read_contract_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/tokens/read_contract_controller.ex index f6ae63f537..2f0c8773a4 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/tokens/read_contract_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/tokens/read_contract_controller.ex @@ -3,17 +3,21 @@ defmodule BlockScoutWeb.Tokens.ReadContractController do alias Explorer.{Chain, Market} + import BlockScoutWeb.Tokens.TokenController, only: [fetch_token_counters: 2] + def index(conn, %{"token_id" => address_hash_string}) do options = [necessity_by_association: %{[contract_address: :smart_contract] => :optional}] with {:ok, address_hash} <- Chain.string_to_address_hash(address_hash_string), {:ok, token} <- Chain.token_from_address_hash(address_hash, options) do + {total_token_transfers, total_token_holders} = fetch_token_counters(token, address_hash) + render( conn, "index.html", token: Market.add_price(token), - total_token_transfers: token.holder_count || Chain.count_token_transfers_from_token_hash(address_hash), - total_token_holders: Chain.count_token_holders_from_token_hash(address_hash) + total_token_transfers: total_token_transfers, + total_token_holders: total_token_holders ) else :error -> diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/tokens/token_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/tokens/token_controller.ex index 47b73cc942..58d6978008 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/tokens/token_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/tokens/token_controller.ex @@ -1,7 +1,41 @@ defmodule BlockScoutWeb.Tokens.TokenController do use BlockScoutWeb, :controller + require Logger + + alias Explorer.Chain + def show(conn, %{"id" => address_hash_string}) do redirect(conn, to: token_transfer_path(conn, :index, address_hash_string)) end + + def fetch_token_counters(token, address_hash) do + total_token_transfers_task = + Task.async(fn -> + Chain.count_token_transfers_from_token_hash(address_hash) + end) + + total_token_holders_task = + Task.async(fn -> + token.holder_count || Chain.count_token_holders_from_token_hash(address_hash) + end) + + [total_token_transfers_task, total_token_holders_task] + |> Task.yield_many(:timer.seconds(40)) + |> Enum.map(fn {_task, res} -> + case res do + {:ok, result} -> + result + + {:exit, reason} -> + Logger.warn("Query fetching token counters terminated: #{inspect(reason)}") + 0 + + nil -> + Logger.warn("Query fetching token counters timed out.") + 0 + end + end) + |> List.to_tuple() + end end diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/tokens/transfer_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/tokens/transfer_controller.ex index 9b75087c45..f977d091f1 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/tokens/transfer_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/tokens/transfer_controller.ex @@ -6,6 +6,7 @@ defmodule BlockScoutWeb.Tokens.TransferController do alias Phoenix.View import BlockScoutWeb.Chain, only: [split_list_by_page: 1, paging_options: 1, next_page_params: 3] + import BlockScoutWeb.Tokens.TokenController, only: [fetch_token_counters: 2] def index(conn, %{"token_id" => address_hash_string, "type" => "JSON"} = params) do with {:ok, address_hash} <- Chain.string_to_address_hash(address_hash_string), @@ -48,13 +49,15 @@ defmodule BlockScoutWeb.Tokens.TransferController do with {:ok, address_hash} <- Chain.string_to_address_hash(address_hash_string), {:ok, token} <- Chain.token_from_address_hash(address_hash, options) do + {total_token_transfers, total_token_holders} = fetch_token_counters(token, address_hash) + render( conn, "index.html", current_path: current_path(conn), token: Market.add_price(token), - total_token_transfers: Chain.count_token_transfers_from_token_hash(address_hash), - total_token_holders: token.holder_count || Chain.count_token_holders_from_token_hash(address_hash) + total_token_transfers: total_token_transfers, + total_token_holders: total_token_holders ) else :error -> diff --git a/apps/block_scout_web/lib/block_scout_web/templates/tokens/overview/_details.html.eex b/apps/block_scout_web/lib/block_scout_web/templates/tokens/overview/_details.html.eex index c4fbdc2af2..8f2928990a 100644 --- a/apps/block_scout_web/lib/block_scout_web/templates/tokens/overview/_details.html.eex +++ b/apps/block_scout_web/lib/block_scout_web/templates/tokens/overview/_details.html.eex @@ -57,8 +57,12 @@
<%= @token.type %> - <%= @total_token_holders %> <%= gettext "Addresses" %> - <%= @total_token_transfers %> <%= gettext "Transfers" %> + <%= if @total_token_holders > 0 do %> + <%= @total_token_holders %> <%= gettext "Addresses" %> + <% end %> + <%= if @total_token_transfers > 0 do %> + <%= @total_token_transfers %> <%= gettext "Transfers" %> + <% end %> <%= if decimals?(@token) do %> <%= @token.decimals %> <%= gettext "Decimals" %> <% end %> From 3969ae24a845f037605bf2a14e49ff626479de07 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Wed, 4 Sep 2019 16:14:07 +0300 Subject: [PATCH 10/67] add CHANGELOG entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f347667b27..9a18b3f439 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## Current ### Features +- [#2666](https://github.com/poanetwork/blockscout/pull/2666) - fetch token counters in parallel - [#2636](https://github.com/poanetwork/blockscout/pull/2636) - Execute all address' transactions page queries in parallel - [#2596](https://github.com/poanetwork/blockscout/pull/2596) - support AuRa's empty step reward type - [#2581](https://github.com/poanetwork/blockscout/pull/2581) - Add generic Map-like Cache behaviour and implementation From 1d2f27e7ca7ae4a2c260031d6a724b722714dc73 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Wed, 4 Sep 2019 16:41:34 +0300 Subject: [PATCH 11/67] fix gettext --- apps/block_scout_web/priv/gettext/default.pot | 14 +++++++------- .../priv/gettext/en/LC_MESSAGES/default.po | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/apps/block_scout_web/priv/gettext/default.pot b/apps/block_scout_web/priv/gettext/default.pot index f34f711620..bef85a6704 100644 --- a/apps/block_scout_web/priv/gettext/default.pot +++ b/apps/block_scout_web/priv/gettext/default.pot @@ -135,7 +135,7 @@ msgstr "" #, elixir-format #: lib/block_scout_web/templates/address/index.html.eex:4 -#: lib/block_scout_web/templates/tokens/overview/_details.html.eex:60 +#: lib/block_scout_web/templates/tokens/overview/_details.html.eex:61 msgid "Addresses" msgstr "" @@ -304,8 +304,8 @@ msgstr "" #: lib/block_scout_web/templates/address/_validator_metadata_modal.html.eex:37 #: lib/block_scout_web/templates/address/overview.html.eex:145 #: lib/block_scout_web/templates/address/overview.html.eex:153 -#: lib/block_scout_web/templates/tokens/overview/_details.html.eex:106 -#: lib/block_scout_web/templates/tokens/overview/_details.html.eex:114 +#: lib/block_scout_web/templates/tokens/overview/_details.html.eex:110 +#: lib/block_scout_web/templates/tokens/overview/_details.html.eex:118 msgid "Close" msgstr "" @@ -526,7 +526,7 @@ msgid "Data" msgstr "" #, elixir-format -#: lib/block_scout_web/templates/tokens/overview/_details.html.eex:63 +#: lib/block_scout_web/templates/tokens/overview/_details.html.eex:67 msgid "Decimals" msgstr "" @@ -1278,7 +1278,7 @@ msgstr "" #: lib/block_scout_web/templates/address/overview.html.eex:33 #: lib/block_scout_web/templates/address/overview.html.eex:144 #: lib/block_scout_web/templates/tokens/overview/_details.html.eex:36 -#: lib/block_scout_web/templates/tokens/overview/_details.html.eex:105 +#: lib/block_scout_web/templates/tokens/overview/_details.html.eex:109 msgid "QR Code" msgstr "" @@ -1555,7 +1555,7 @@ msgid "Topics" msgstr "" #, elixir-format -#: lib/block_scout_web/templates/tokens/overview/_details.html.eex:75 +#: lib/block_scout_web/templates/tokens/overview/_details.html.eex:79 msgid "Total Supply" msgstr "" @@ -1612,7 +1612,7 @@ msgid "Transactions sent" msgstr "" #, elixir-format -#: lib/block_scout_web/templates/tokens/overview/_details.html.eex:61 +#: lib/block_scout_web/templates/tokens/overview/_details.html.eex:64 msgid "Transfers" msgstr "" diff --git a/apps/block_scout_web/priv/gettext/en/LC_MESSAGES/default.po b/apps/block_scout_web/priv/gettext/en/LC_MESSAGES/default.po index f34f711620..bef85a6704 100644 --- a/apps/block_scout_web/priv/gettext/en/LC_MESSAGES/default.po +++ b/apps/block_scout_web/priv/gettext/en/LC_MESSAGES/default.po @@ -135,7 +135,7 @@ msgstr "" #, elixir-format #: lib/block_scout_web/templates/address/index.html.eex:4 -#: lib/block_scout_web/templates/tokens/overview/_details.html.eex:60 +#: lib/block_scout_web/templates/tokens/overview/_details.html.eex:61 msgid "Addresses" msgstr "" @@ -304,8 +304,8 @@ msgstr "" #: lib/block_scout_web/templates/address/_validator_metadata_modal.html.eex:37 #: lib/block_scout_web/templates/address/overview.html.eex:145 #: lib/block_scout_web/templates/address/overview.html.eex:153 -#: lib/block_scout_web/templates/tokens/overview/_details.html.eex:106 -#: lib/block_scout_web/templates/tokens/overview/_details.html.eex:114 +#: lib/block_scout_web/templates/tokens/overview/_details.html.eex:110 +#: lib/block_scout_web/templates/tokens/overview/_details.html.eex:118 msgid "Close" msgstr "" @@ -526,7 +526,7 @@ msgid "Data" msgstr "" #, elixir-format -#: lib/block_scout_web/templates/tokens/overview/_details.html.eex:63 +#: lib/block_scout_web/templates/tokens/overview/_details.html.eex:67 msgid "Decimals" msgstr "" @@ -1278,7 +1278,7 @@ msgstr "" #: lib/block_scout_web/templates/address/overview.html.eex:33 #: lib/block_scout_web/templates/address/overview.html.eex:144 #: lib/block_scout_web/templates/tokens/overview/_details.html.eex:36 -#: lib/block_scout_web/templates/tokens/overview/_details.html.eex:105 +#: lib/block_scout_web/templates/tokens/overview/_details.html.eex:109 msgid "QR Code" msgstr "" @@ -1555,7 +1555,7 @@ msgid "Topics" msgstr "" #, elixir-format -#: lib/block_scout_web/templates/tokens/overview/_details.html.eex:75 +#: lib/block_scout_web/templates/tokens/overview/_details.html.eex:79 msgid "Total Supply" msgstr "" @@ -1612,7 +1612,7 @@ msgid "Transactions sent" msgstr "" #, elixir-format -#: lib/block_scout_web/templates/tokens/overview/_details.html.eex:61 +#: lib/block_scout_web/templates/tokens/overview/_details.html.eex:64 msgid "Transfers" msgstr "" From b1ac8c69ef165380e2f368f9d7401cc5445e129d Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Fri, 6 Sep 2019 11:38:50 +0300 Subject: [PATCH 12/67] add test for all supported chains --- .../exchange_rates/source/coin_gecko_test.exs | 66 ++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/apps/explorer/test/explorer/exchange_rates/source/coin_gecko_test.exs b/apps/explorer/test/explorer/exchange_rates/source/coin_gecko_test.exs index 3072099cbc..76d210cd02 100644 --- a/apps/explorer/test/explorer/exchange_rates/source/coin_gecko_test.exs +++ b/apps/explorer/test/explorer/exchange_rates/source/coin_gecko_test.exs @@ -34,6 +34,26 @@ defmodule Explorer.ExchangeRates.Source.CoinGeckoTest do "id": "pocket-arena", "symbol": "poc", "name": "Pocket Arena" + }, + { + "id": "ethereum", + "symbol": "eth", + "name": "Ethereum" + }, + { + "id": "rootstock", + "symbol": "rbtc", + "name": "Rootstock RSK" + }, + { + "id": "dai", + "symbol": "dai", + "name": "Dai" + }, + { + "id": "callisto", + "symbol": "clo", + "name": "Callisto Network" } ] """ @@ -88,15 +108,59 @@ defmodule Explorer.ExchangeRates.Source.CoinGeckoTest do bypass = Bypass.open() Application.put_env(:explorer, CoinGecko, base_url: "http://localhost:#{bypass.port}") + on_exit(fn -> + Application.put_env(:explorer, :coin, "POA") + end) + {:ok, bypass: bypass} end - test "fetches coin id", %{bypass: bypass} do + test "fetches poa coin id by default", %{bypass: bypass} do Bypass.expect(bypass, "GET", "/coins/list", fn conn -> Conn.resp(conn, 200, @coins_list) end) assert CoinGecko.coin_id() == {:ok, "poa-network"} end + + test "fetches eth coin id", %{bypass: bypass} do + Application.put_env(:explorer, :coin, "ETH") + + Bypass.expect(bypass, "GET", "/coins/list", fn conn -> + Conn.resp(conn, 200, @coins_list) + end) + + assert CoinGecko.coin_id() == {:ok, "ethereum"} + end + + test "fetches rbtc coin id", %{bypass: bypass} do + Application.put_env(:explorer, :coin, "RBTC") + + Bypass.expect(bypass, "GET", "/coins/list", fn conn -> + Conn.resp(conn, 200, @coins_list) + end) + + assert CoinGecko.coin_id() == {:ok, "rootstock"} + end + + test "fetches dai coin id", %{bypass: bypass} do + Application.put_env(:explorer, :coin, "DAI") + + Bypass.expect(bypass, "GET", "/coins/list", fn conn -> + Conn.resp(conn, 200, @coins_list) + end) + + assert CoinGecko.coin_id() == {:ok, "dai"} + end + + test "fetches callisto coin id", %{bypass: bypass} do + Application.put_env(:explorer, :coin, "CLO") + + Bypass.expect(bypass, "GET", "/coins/list", fn conn -> + Conn.resp(conn, 200, @coins_list) + end) + + assert CoinGecko.coin_id() == {:ok, "callisto"} + end end end From d7f7ad64cc856bbc0ff9656c4a9109a7b4963ada Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Fri, 6 Sep 2019 12:15:25 +0300 Subject: [PATCH 13/67] fix typo --- apps/explorer/lib/explorer/exchange_rates/source/coin_gecko.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/explorer/lib/explorer/exchange_rates/source/coin_gecko.ex b/apps/explorer/lib/explorer/exchange_rates/source/coin_gecko.ex index 5a6199acb2..9d4cfba3f4 100644 --- a/apps/explorer/lib/explorer/exchange_rates/source/coin_gecko.ex +++ b/apps/explorer/lib/explorer/exchange_rates/source/coin_gecko.ex @@ -70,7 +70,7 @@ defmodule Explorer.ExchangeRates.Source.CoinGecko do if symbol_data do {:ok, symbol_data["id"]} else - {:errpr, :not_found} + {:error, :not_found} end {:ok, %Response{body: body, status_code: status_code}} when status_code in 400..499 -> From f22a7a3f268eefbe5d8fb10516ab90f0bc750baf Mon Sep 17 00:00:00 2001 From: Yegor Date: Sat, 7 Sep 2019 23:59:15 +0300 Subject: [PATCH 14/67] Update CHANGELOG.md --- CHANGELOG.md | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e345f9216d..8199bc9cd3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,20 +1,39 @@ ## Current ### Features -- [#2555](https://github.com/poanetwork/blockscout/pull/2555) - find and show decoding candidates for logs +- [#2671](https://github.com/poanetwork/blockscout/pull/2671) - fixed buttons color at smart contract section +- [#2672](https://github.com/poanetwork/blockscout/pull/2672) - added new theme for xUSDT + +### Fixes + +### Chore + + +## 2.0.4-beta + +### Features +- [#2636](https://github.com/poanetwork/blockscout/pull/2636) - Execute all address' transactions page queries in parallel - [#2596](https://github.com/poanetwork/blockscout/pull/2596) - support AuRa's empty step reward type +- [#2588](https://github.com/poanetwork/blockscout/pull/2588) - add verification submission comment +- [#2505](https://github.com/poanetwork/blockscout/pull/2505) - support POA Network emission rewards +- [#2581](https://github.com/poanetwork/blockscout/pull/2581) - Add generic Map-like Cache behaviour and implementation - [#2561](https://github.com/poanetwork/blockscout/pull/2561) - Add token's type to the response of tokenlist method +- [#2555](https://github.com/poanetwork/blockscout/pull/2555) - find and show decoding candidates for logs - [#2499](https://github.com/poanetwork/blockscout/pull/2499) - import emission reward ranges - [#2497](https://github.com/poanetwork/blockscout/pull/2497) - Add generic Ordered Cache behaviour and implementation ### Fixes +- [#2659](https://github.com/poanetwork/blockscout/pull/2659) - Multipurpose front-end part update - [#2640](https://github.com/poanetwork/blockscout/pull/2640) - SVG network icons - [#2635](https://github.com/poanetwork/blockscout/pull/2635) - optimize ERC721 inventory query - [#2626](https://github.com/poanetwork/blockscout/pull/2626) - Fixing 2 Mobile UI Issues - [#2623](https://github.com/poanetwork/blockscout/pull/2623) - fix a blinking test - [#2616](https://github.com/poanetwork/blockscout/pull/2616) - deduplicate coin history records by delta - [#2613](https://github.com/poanetwork/blockscout/pull/2613) - fix getminedblocks rpc endpoint +- [#2612](https://github.com/poanetwork/blockscout/pull/2612) - Add cache updating independently from Indexer +- [#2610](https://github.com/poanetwork/blockscout/pull/2610) - use CoinGecko instead of CoinMarketcap for exchange rates - [#2592](https://github.com/poanetwork/blockscout/pull/2592) - process new metadata format for whisper +- [#2591](https://github.com/poanetwork/blockscout/pull/2591) - Fix url error in API page - [#2572](https://github.com/poanetwork/blockscout/pull/2572) - Ease non-critical css - [#2570](https://github.com/poanetwork/blockscout/pull/2570) - Network icons preload - [#2569](https://github.com/poanetwork/blockscout/pull/2569) - do not fetch emission rewards for transactions csv exporter @@ -22,11 +41,13 @@ - [#2564](https://github.com/poanetwork/blockscout/pull/2564) - fix first page button for uncles and reorgs - [#2563](https://github.com/poanetwork/blockscout/pull/2563) - Fix view less transfers button - [#2538](https://github.com/poanetwork/blockscout/pull/2538) - fetch the last not empty coin balance records +- [#2468](https://github.com/poanetwork/blockscout/pull/2468) - fix confirmations for non consensus blocks ### Chore +- [#2646](https://github.com/poanetwork/blockscout/pull/2646) - Added Xerom to list of Additional Chains using BlockScout - [#2634](https://github.com/poanetwork/blockscout/pull/2634) - add Lukso to networks dropdown -- [#2611](https://github.com/poanetwork/blockscout/pull/2611) - fix js dependency vulnerabilities - [#2617](https://github.com/poanetwork/blockscout/pull/2617) - skip cache update if there are no blocks inserted +- [#2611](https://github.com/poanetwork/blockscout/pull/2611) - fix js dependency vulnerabilities - [#2594](https://github.com/poanetwork/blockscout/pull/2594) - do not start genesis data fetching periodically - [#2590](https://github.com/poanetwork/blockscout/pull/2590) - restore backward compatablity with old releases - [#2577](https://github.com/poanetwork/blockscout/pull/2577) - Need recompile column in the env vars table From ad1f1c1826f2f955142d7362d6377886eaa2b395 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Mon, 9 Sep 2019 12:47:08 +0300 Subject: [PATCH 15/67] Revert "Filter pending logs" --- CHANGELOG.md | 1 - .../lib/ethereum_jsonrpc/receipts.ex | 4 +--- .../test/indexer/block/fetcher/receipts_test.exs | 14 -------------- 3 files changed, 1 insertion(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 06ce0a42b7..b3f20f4061 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -232,7 +232,6 @@ - [#2123](https://github.com/poanetwork/blockscout/pull/2123) - fix coins percentage view - [#2119](https://github.com/poanetwork/blockscout/pull/2119) - fix map logging - [#2130](https://github.com/poanetwork/blockscout/pull/2130) - fix navigation -- [#2148](https://github.com/poanetwork/blockscout/pull/2148) - filter pending logs - [#2147](https://github.com/poanetwork/blockscout/pull/2147) - add rsk format of checksum - [#2149](https://github.com/poanetwork/blockscout/pull/2149) - remove pending transaction count - [#2177](https://github.com/poanetwork/blockscout/pull/2177) - remove duplicate entries from UncleBlock's Fetcher diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/receipts.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/receipts.ex index 7028423f7a..e5d706589c 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/receipts.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/receipts.ex @@ -61,9 +61,7 @@ defmodule EthereumJSONRPC.Receipts do """ @spec elixir_to_logs(elixir) :: Logs.elixir() def elixir_to_logs(elixir) when is_list(elixir) do - elixir - |> Enum.flat_map(&Receipt.elixir_to_logs/1) - |> Enum.filter(&(Map.get(&1, "type") != "pending")) + Enum.flat_map(elixir, &Receipt.elixir_to_logs/1) end @doc """ diff --git a/apps/indexer/test/indexer/block/fetcher/receipts_test.exs b/apps/indexer/test/indexer/block/fetcher/receipts_test.exs index 35371b7c4a..95570db4a2 100644 --- a/apps/indexer/test/indexer/block/fetcher/receipts_test.exs +++ b/apps/indexer/test/indexer/block/fetcher/receipts_test.exs @@ -84,18 +84,6 @@ defmodule Indexer.Block.Fetcher.ReceiptsTest do "transactionIndex" => "0x0", "transactionLogIndex" => "0x0", "type" => "mined" - }, - %{ - "address" => "0x8bf38d4764929064f2d4d3a56520a76ab3df415c", - "blockHash" => nil, - "blockNumber" => nil, - "data" => "0x000000000000000000000000862d67cb0773ee3f8ce7ea89b328ffea861ab3ef", - "logIndex" => "0x1", - "topics" => ["0x600bcf04a13e752d1e3670a5a9f1c21177ca2a93c6f5391d4f1298d098097c22"], - "transactionHash" => "0x53bd884872de3e488692881baeec262e7b95234d3965248c39fe992fffd433e5", - "transactionIndex" => "0x0", - "transactionLogIndex" => "0x0", - "type" => "pending" } ], "logsBloom" => @@ -158,8 +146,6 @@ defmodule Indexer.Block.Fetcher.ReceiptsTest do log[:transaction_hash] == "0x43bd884872de3e488692881baeec262e7b95234d3965248c39fe992fffd433e5" && log[:block_number] == 46147 end) - - refute Enum.find(logs, fn log -> log[:type] == "pending" end) end end end From da583904e515a8a83f95f7651dbe0a502e54e255 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Mon, 9 Sep 2019 16:28:14 +0300 Subject: [PATCH 16/67] add CHANGELOG entry --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3f20f4061..d719db99bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,9 +2,10 @@ ### Features - [#2678](https://github.com/poanetwork/blockscout/pull/2678) - fixed dashboard banner height bug -- [#2672](https://github.com/poanetwork/blockscout/pull/2672) - added new theme for xUSDT +- [#2672](https://github.com/poanetwork/blockscout/pull/2672) - added new theme for xUSDT ### Fixes +- [#2684](https://github.com/poanetwork/blockscout/pull/2684) - do not filter pending logs ### Chore @@ -232,6 +233,7 @@ - [#2123](https://github.com/poanetwork/blockscout/pull/2123) - fix coins percentage view - [#2119](https://github.com/poanetwork/blockscout/pull/2119) - fix map logging - [#2130](https://github.com/poanetwork/blockscout/pull/2130) - fix navigation +- [#2148](https://github.com/poanetwork/blockscout/pull/2148) - filter pending logs - [#2147](https://github.com/poanetwork/blockscout/pull/2147) - add rsk format of checksum - [#2149](https://github.com/poanetwork/blockscout/pull/2149) - remove pending transaction count - [#2177](https://github.com/poanetwork/blockscout/pull/2177) - remove duplicate entries from UncleBlock's Fetcher From 614369154ec5c80ae0fdc01780e894ca652b9230 Mon Sep 17 00:00:00 2001 From: Yegor Date: Tue, 10 Sep 2019 12:07:10 +0300 Subject: [PATCH 17/67] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 343301eb65..f5588f9e62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## Current ### Features +- [#2671](https://github.com/poanetwork/blockscout/pull/2671) - fixed buttons color at smart contract section - [#2679](https://github.com/poanetwork/blockscout/pull/2679) - added fixed height for card chain blocks and card chain transactions - [#2678](https://github.com/poanetwork/blockscout/pull/2678) - fixed dashboard banner height bug - [#2672](https://github.com/poanetwork/blockscout/pull/2672) - added new theme for xUSDT From 5aa8d5c456241bba041f00cbb23a6dcb37ae0149 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 10 Sep 2019 12:07:16 +0300 Subject: [PATCH 18/67] remove nonconsensus token transfers --- .../explorer/chain/import/runner/blocks.ex | 55 ++++++++++++++- .../chain/import/runner/blocks_test.exs | 68 +++++++++++++++---- 2 files changed, 110 insertions(+), 13 deletions(-) diff --git a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex index 209d5d2fa6..3c949270cf 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex @@ -9,7 +9,7 @@ defmodule Explorer.Chain.Import.Runner.Blocks do alias Ecto.Adapters.SQL alias Ecto.{Changeset, Multi, Repo} - alias Explorer.Chain.{Address, Block, Hash, Import, InternalTransaction, Transaction} + alias Explorer.Chain.{Address, Block, Hash, Import, InternalTransaction, Transaction, TokenTransfer} alias Explorer.Chain.Block.Reward alias Explorer.Chain.Import.Runner alias Explorer.Chain.Import.Runner.Address.CurrentTokenBalances @@ -73,6 +73,21 @@ defmodule Explorer.Chain.Import.Runner.Blocks do |> Multi.run(:lose_invalid_neighbour_consensus, fn repo, _ -> lose_invalid_neighbour_consensus(repo, where_invalid_neighbour, insert_options) end) + |> Multi.run(:remove_nonconsensus_data, fn repo, + %{ + lose_consensus: lost_consensus_blocks, + lose_invalid_neighbour_consensus: lost_consensus_neighbours + } -> + nonconsensus_block_numbers = + lost_consensus_blocks + |> Kernel.++(lost_consensus_neighbours) + |> Enum.map(fn %{number: number} -> + number + end) + |> Enum.sort() + + remove_nonconsensus_data(repo, nonconsensus_block_numbers, insert_options) + end) |> Multi.run(:delete_address_token_balances, fn repo, _ -> delete_address_token_balances(repo, ordered_consensus_block_numbers, insert_options) end) @@ -342,6 +357,44 @@ defmodule Explorer.Chain.Import.Runner.Blocks do end end + defp remove_nonconsensus_data(repo, nonconsensus_block_numbers, insert_options) do + with {:ok, deleted_token_transfers} <- + remove_nonconsensus_token_transfers(repo, nonconsensus_block_numbers, insert_options) do + {:ok, %{token_transfers: deleted_token_transfers}} + end + end + + defp remove_nonconsensus_token_transfers(repo, nonconsensus_block_numbers, %{timeout: timeout}) do + ordered_token_transfers = + from(token_transfer in TokenTransfer, + where: token_transfer.block_number in ^nonconsensus_block_numbers, + select: map(token_transfer, [:transaction_hash, :log_index]), + order_by: [ + token_transfer.transaction_hash, + token_transfer.log_index + ], + lock: "FOR UPDATE" + ) + + query = + from(token_transfer in TokenTransfer, + select: map(token_transfer, [:transaction_hash, :log_index]), + inner_join: ordered_token_transfer in subquery(ordered_token_transfers), + on: + ordered_token_transfer.transaction_hash == + token_transfer.transaction_hash and + ordered_token_transfer.log_index == token_transfer.log_index + ) + + try do + {_count, deleted_token_transfers} = repo.delete_all(query, timeout: timeout) + + {:ok, deleted_token_transfers} + rescue + postgrex_error in Postgrex.Error -> + {:error, %{exception: postgrex_error, block_numbers: nonconsensus_block_numbers}} + end + end defp delete_address_token_balances(_, [], _), do: {:ok, []} defp delete_address_token_balances(repo, ordered_consensus_block_numbers, %{timeout: timeout}) do diff --git a/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs b/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs index 63110cba40..f1ebaa4c77 100644 --- a/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs +++ b/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs @@ -7,13 +7,14 @@ defmodule Explorer.Chain.Import.Runner.BlocksTest do alias Ecto.Multi alias Explorer.Chain.Import.Runner.{Blocks, Transactions} - alias Explorer.Chain.{Address, Block, Transaction} + alias Explorer.Chain.{Address, Block, Transaction, TokenTransfer} alias Explorer.Chain alias Explorer.Repo describe "run/1" do setup do - block = insert(:block, consensus: true) + miner = insert(:address) + block = params_for(:block, consensus: true, miner_hash: miner.hash) timestamp = DateTime.utc_now() options = %{timestamps: %{inserted_at: timestamp, updated_at: timestamp}} @@ -22,9 +23,11 @@ defmodule Explorer.Chain.Import.Runner.BlocksTest do end test "derive_transaction_forks replaces hash on conflicting (uncle_hash, index)", %{ - consensus_block: %Block{hash: block_hash, miner_hash: miner_hash, number: block_number} = consensus_block, + consensus_block: %{hash: block_hash, miner_hash: miner_hash, number: block_number}, options: options } do + consensus_block = insert(:block, %{hash: block_hash, number: block_number}) + transaction = :transaction |> insert() @@ -81,7 +84,7 @@ defmodule Explorer.Chain.Import.Runner.BlocksTest do end test "delete_address_current_token_balances deletes rows with matching block number when consensus is true", - %{consensus_block: %Block{number: block_number} = block, options: options} do + %{consensus_block: %{number: block_number} = block, options: options} do %Address.CurrentTokenBalance{address_hash: address_hash, token_contract_address_hash: token_contract_address_hash} = insert(:address_current_token_balance, block_number: block_number) @@ -98,7 +101,7 @@ defmodule Explorer.Chain.Import.Runner.BlocksTest do end test "delete_address_current_token_balances does not delete rows with matching block number when consensus is false", - %{consensus_block: %Block{number: block_number} = block, options: options} do + %{consensus_block: %{number: block_number} = block, options: options} do %Address.CurrentTokenBalance{} = insert(:address_current_token_balance, block_number: block_number) count = 1 @@ -113,8 +116,47 @@ defmodule Explorer.Chain.Import.Runner.BlocksTest do assert count(Address.CurrentTokenBalance) == count end + test "remove_nonconsensus_data deletes rows with matching block number when new consensus block is inserted", + %{consensus_block: %{number: block_number} = block, options: options} do + insert(:block, number: block_number, consensus: true) + + %TokenTransfer{transaction_hash: transaction_hash, log_index: log_index} = + insert(:token_transfer, block_number: block_number, transaction: insert(:transaction)) + + assert count(TokenTransfer) == 1 + + assert {:ok, + %{ + remove_nonconsensus_data: %{ + token_transfers: [ + %{transaction_hash: ^transaction_hash, log_index: ^log_index} + ] + } + }} = run_block_consensus_change(block, true, options) + + assert count(TokenTransfer) == 0 + end + + test "delete_token_transfers does not delete rows with matching block number when consensus is false", + %{consensus_block: %{number: block_number} = block, options: options} do + insert(:token_transfer, block_number: block_number, transaction: insert(:transaction)) + + count = 1 + + assert count(TokenTransfer) == count + + assert {:ok, + %{ + remove_nonconsensus_data: %{ + token_transfers: [] + } + }} = run_block_consensus_change(block, false, options) + + assert count(TokenTransfer) == count + end + test "derive_address_current_token_balances inserts rows if there is an address_token_balance left for the rows deleted by delete_address_current_token_balances", - %{consensus_block: %Block{number: block_number} = block, options: options} do + %{consensus_block: %{number: block_number} = block, options: options} do token = insert(:token) token_contract_address_hash = token.contract_address_hash @@ -172,7 +214,7 @@ defmodule Explorer.Chain.Import.Runner.BlocksTest do end test "a non-holder reverting to a holder increases the holder_count", - %{consensus_block: %Block{hash: block_hash, miner_hash: miner_hash, number: block_number}, options: options} do + %{consensus_block: %{hash: block_hash, miner_hash: miner_hash, number: block_number}, options: options} do token = insert(:token) token_contract_address_hash = token.contract_address_hash @@ -204,7 +246,7 @@ defmodule Explorer.Chain.Import.Runner.BlocksTest do end test "a holder reverting to a non-holder decreases the holder_count", - %{consensus_block: %Block{hash: block_hash, miner_hash: miner_hash, number: block_number}, options: options} do + %{consensus_block: %{hash: block_hash, miner_hash: miner_hash, number: block_number}, options: options} do token = insert(:token) token_contract_address_hash = token.contract_address_hash @@ -236,7 +278,7 @@ defmodule Explorer.Chain.Import.Runner.BlocksTest do end test "a non-holder becoming and a holder becoming while a holder becomes a non-holder cancels out and holder_count does not change", - %{consensus_block: %Block{number: block_number} = block, options: options} do + %{consensus_block: %{number: block_number} = block, options: options} do token = insert(:token) token_contract_address_hash = token.contract_address_hash @@ -262,7 +304,8 @@ defmodule Explorer.Chain.Import.Runner.BlocksTest do # Regression test for https://github.com/poanetwork/blockscout/issues/1644 test "discards neighbouring blocks if they aren't related to the current one because of reorg and/or import timeout", - %{consensus_block: %Block{number: block_number, hash: block_hash, miner_hash: miner_hash}, options: options} do + %{consensus_block: %{number: block_number, hash: block_hash, miner_hash: miner_hash}, options: options} do + insert(:block, %{number: block_number, hash: block_hash}) old_block1 = params_for(:block, miner_hash: miner_hash, parent_hash: block_hash, number: block_number + 1) new_block1 = params_for(:block, miner_hash: miner_hash, parent_hash: block_hash, number: block_number + 1) @@ -286,7 +329,8 @@ defmodule Explorer.Chain.Import.Runner.BlocksTest do # Regression test for https://github.com/poanetwork/blockscout/issues/1911 test "forces block refetch if transaction is re-collated in a different block", - %{consensus_block: %Block{number: block_number, hash: block_hash, miner_hash: miner_hash}, options: options} do + %{consensus_block: %{number: block_number, hash: block_hash, miner_hash: miner_hash}, options: options} do + insert(:block, %{number: block_number, hash: block_hash}) new_block1 = params_for(:block, miner_hash: miner_hash, parent_hash: block_hash, number: block_number + 1) new_block2 = params_for(:block, miner_hash: miner_hash, parent_hash: new_block1.hash, number: block_number + 2) @@ -365,7 +409,7 @@ defmodule Explorer.Chain.Import.Runner.BlocksTest do end defp run_block_consensus_change( - %Block{hash: block_hash, miner_hash: miner_hash, number: block_number}, + %{hash: block_hash, miner_hash: miner_hash, number: block_number}, consensus, options ) do From 0afd72617b8a5652667ac61da58d381cbbd5ab39 Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Tue, 10 Sep 2019 12:24:10 +0300 Subject: [PATCH 19/67] Update CHANGELOG.md --- CHANGELOG.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c2965b4395..eb56373651 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,12 +4,11 @@ - [#2679](https://github.com/poanetwork/blockscout/pull/2679) - added fixed height for card chain blocks and card chain transactions - [#2678](https://github.com/poanetwork/blockscout/pull/2678) - fixed dashboard banner height bug - [#2672](https://github.com/poanetwork/blockscout/pull/2672) - added new theme for xUSDT +- [#2663](https://github.com/poanetwork/blockscout/pull/2663) - Fetch address counters in parallel ### Fixes - [#2684](https://github.com/poanetwork/blockscout/pull/2684) - do not filter pending logs - [#2682](https://github.com/poanetwork/blockscout/pull/2682) - Use Task.start instead of Task.async in caches -- [#2663](https://github.com/poanetwork/blockscout/pull/2663) - Fetch address counters in parallel - ### Chore From 934af7604ef8cf04680a4b1f80e05cad7c16808d Mon Sep 17 00:00:00 2001 From: Yegor Date: Tue, 10 Sep 2019 12:54:09 +0300 Subject: [PATCH 20/67] Update CHANGELOG.md --- CHANGELOG.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5588f9e62..1e08951e9e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,6 @@ ## Current ### Features -- [#2671](https://github.com/poanetwork/blockscout/pull/2671) - fixed buttons color at smart contract section - [#2679](https://github.com/poanetwork/blockscout/pull/2679) - added fixed height for card chain blocks and card chain transactions - [#2678](https://github.com/poanetwork/blockscout/pull/2678) - fixed dashboard banner height bug - [#2672](https://github.com/poanetwork/blockscout/pull/2672) - added new theme for xUSDT @@ -9,7 +8,7 @@ ### Fixes - [#2682](https://github.com/poanetwork/blockscout/pull/2682) - Use Task.start instead of Task.async in caches - +- [#2671](https://github.com/poanetwork/blockscout/pull/2671) - fixed buttons color at smart contract section ### Chore From c195428634d9d2f917ee4f21f1dc0fde36023b62 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 10 Sep 2019 13:04:15 +0300 Subject: [PATCH 21/67] remove nonconsensus logs --- .../explorer/chain/import/runner/blocks.ex | 79 +++++++++++++++---- .../chain/import/runner/blocks_test.exs | 28 ++++++- 2 files changed, 88 insertions(+), 19 deletions(-) diff --git a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex index 3c949270cf..5c4c42d898 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex @@ -9,7 +9,7 @@ defmodule Explorer.Chain.Import.Runner.Blocks do alias Ecto.Adapters.SQL alias Ecto.{Changeset, Multi, Repo} - alias Explorer.Chain.{Address, Block, Hash, Import, InternalTransaction, Transaction, TokenTransfer} + alias Explorer.Chain.{Address, Block, Hash, Import, InternalTransaction, Log, TokenTransfer, Transaction} alias Explorer.Chain.Block.Reward alias Explorer.Chain.Import.Runner alias Explorer.Chain.Import.Runner.Address.CurrentTokenBalances @@ -58,15 +58,6 @@ defmodule Explorer.Chain.Import.Runner.Blocks do where_forked: where_forked }) end) - # MUST be after `:derive_transaction_forks`, which depends on values in `transactions` table - |> Multi.run(:fork_transactions, fn repo, _ -> - fork_transactions(%{ - repo: repo, - timeout: options[Runner.Transactions.option_key()][:timeout] || Runner.Transactions.timeout(), - timestamps: timestamps, - where_forked: where_forked - }) - end) |> Multi.run(:lose_consensus, fn repo, _ -> lose_consensus(repo, ordered_consensus_block_numbers, insert_options) end) @@ -86,7 +77,20 @@ defmodule Explorer.Chain.Import.Runner.Blocks do end) |> Enum.sort() - remove_nonconsensus_data(repo, nonconsensus_block_numbers, insert_options) + remove_nonconsensus_data( + repo, + nonconsensus_block_numbers, + insert_options + ) + end) + # MUST be after `:derive_transaction_forks`, which depends on values in `transactions` table + |> Multi.run(:fork_transactions, fn repo, _ -> + fork_transactions(%{ + repo: repo, + timeout: options[Runner.Transactions.option_key()][:timeout] || Runner.Transactions.timeout(), + timestamps: timestamps, + where_forked: where_forked + }) end) |> Multi.run(:delete_address_token_balances, fn repo, _ -> delete_address_token_balances(repo, ordered_consensus_block_numbers, insert_options) @@ -357,10 +361,15 @@ defmodule Explorer.Chain.Import.Runner.Blocks do end end - defp remove_nonconsensus_data(repo, nonconsensus_block_numbers, insert_options) do + defp remove_nonconsensus_data( + repo, + nonconsensus_block_numbers, + insert_options + ) do with {:ok, deleted_token_transfers} <- - remove_nonconsensus_token_transfers(repo, nonconsensus_block_numbers, insert_options) do - {:ok, %{token_transfers: deleted_token_transfers}} + remove_nonconsensus_token_transfers(repo, nonconsensus_block_numbers, insert_options), + {:ok, deleted_logs} <- remove_nonconsensus_logs(repo, nonconsensus_block_numbers, insert_options) do + {:ok, %{token_transfers: deleted_token_transfers, logs: deleted_logs}} end end @@ -381,8 +390,8 @@ defmodule Explorer.Chain.Import.Runner.Blocks do select: map(token_transfer, [:transaction_hash, :log_index]), inner_join: ordered_token_transfer in subquery(ordered_token_transfers), on: - ordered_token_transfer.transaction_hash == - token_transfer.transaction_hash and + ordered_token_transfer.transaction_hash == + token_transfer.transaction_hash and ordered_token_transfer.log_index == token_transfer.log_index ) @@ -395,6 +404,44 @@ defmodule Explorer.Chain.Import.Runner.Blocks do {:error, %{exception: postgrex_error, block_numbers: nonconsensus_block_numbers}} end end + + defp remove_nonconsensus_logs(repo, nonconsensus_block_numbers, %{timeout: timeout}) do + transaction_query = + from(transaction in Transaction, + where: transaction.block_number in ^nonconsensus_block_numbers, + select: map(transaction, [:hash]), + order_by: transaction.hash + ) + + ordered_logs = + from(log in Log, + inner_join: transaction in subquery(transaction_query), + on: log.transaction_hash == transaction.hash, + select: map(log, [:transaction_hash, :index]), + order_by: [ + log.transaction_hash, + log.index + ], + lock: "FOR UPDATE" + ) + + query = + from(log in Log, + select: map(log, [:transaction_hash, :index]), + inner_join: ordered_log in subquery(ordered_logs), + on: ordered_log.transaction_hash == log.transaction_hash and ordered_log.index == log.index + ) + + try do + {_count, deleted_logs} = repo.delete_all(query, timeout: timeout) + + {:ok, deleted_logs} + rescue + postgrex_error in Postgrex.Error -> + {:error, %{exception: postgrex_error, block_numbers: nonconsensus_block_numbers}} + end + end + defp delete_address_token_balances(_, [], _), do: {:ok, []} defp delete_address_token_balances(repo, ordered_consensus_block_numbers, %{timeout: timeout}) do diff --git a/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs b/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs index f1ebaa4c77..459363839b 100644 --- a/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs +++ b/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs @@ -7,7 +7,7 @@ defmodule Explorer.Chain.Import.Runner.BlocksTest do alias Ecto.Multi alias Explorer.Chain.Import.Runner.{Blocks, Transactions} - alias Explorer.Chain.{Address, Block, Transaction, TokenTransfer} + alias Explorer.Chain.{Address, Block, Log, Transaction, TokenTransfer} alias Explorer.Chain alias Explorer.Repo @@ -116,7 +116,7 @@ defmodule Explorer.Chain.Import.Runner.BlocksTest do assert count(Address.CurrentTokenBalance) == count end - test "remove_nonconsensus_data deletes rows with matching block number when new consensus block is inserted", + test "remove_nonconsensus_data deletes token transfer rows with matching block number when new consensus block is inserted", %{consensus_block: %{number: block_number} = block, options: options} do insert(:block, number: block_number, consensus: true) @@ -137,7 +137,7 @@ defmodule Explorer.Chain.Import.Runner.BlocksTest do assert count(TokenTransfer) == 0 end - test "delete_token_transfers does not delete rows with matching block number when consensus is false", + test "remove_nonconsensus_data does not delete token transfer rows with matching block number when new consensus block wasn't inserted", %{consensus_block: %{number: block_number} = block, options: options} do insert(:token_transfer, block_number: block_number, transaction: insert(:transaction)) @@ -155,6 +155,28 @@ defmodule Explorer.Chain.Import.Runner.BlocksTest do assert count(TokenTransfer) == count end + test "remove_nonconsensus_data deletes nonconsensus logs", %{ + consensus_block: %{number: block_number} = block, + options: options + } do + old_block = insert(:block, number: block_number, consensus: true) + forked_transaction = :transaction |> insert() |> with_block(old_block) + %Log{transaction_hash: hash, index: index} = insert(:log, transaction: forked_transaction) + + assert count(Log) == 1 + + assert {:ok, + %{ + remove_nonconsensus_data: %{ + logs: [ + %{transaction_hash: ^hash, index: ^index} + ] + } + }} = run_block_consensus_change(block, true, options) + + assert count(Log) == 0 + end + test "derive_address_current_token_balances inserts rows if there is an address_token_balance left for the rows deleted by delete_address_current_token_balances", %{consensus_block: %{number: block_number} = block, options: options} do token = insert(:token) From ab655103f60ccce8427224e5649d576b9671ba29 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 10 Sep 2019 13:11:13 +0300 Subject: [PATCH 22/67] use existing forked transactions query --- .../lib/explorer/chain/import/runner/blocks.ex | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex index 5c4c42d898..cbf7532ca2 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex @@ -79,7 +79,7 @@ defmodule Explorer.Chain.Import.Runner.Blocks do remove_nonconsensus_data( repo, - nonconsensus_block_numbers, + %{nonconsensus_block_numbers: nonconsensus_block_numbers, forked_transactions_query: where_forked}, insert_options ) end) @@ -363,12 +363,15 @@ defmodule Explorer.Chain.Import.Runner.Blocks do defp remove_nonconsensus_data( repo, - nonconsensus_block_numbers, + %{ + nonconsensus_block_numbers: nonconsensus_block_numbers, + forked_transactions_query: forked_transactions_query + }, insert_options ) do with {:ok, deleted_token_transfers} <- remove_nonconsensus_token_transfers(repo, nonconsensus_block_numbers, insert_options), - {:ok, deleted_logs} <- remove_nonconsensus_logs(repo, nonconsensus_block_numbers, insert_options) do + {:ok, deleted_logs} <- remove_nonconsensus_logs(repo, forked_transactions_query, insert_options) do {:ok, %{token_transfers: deleted_token_transfers, logs: deleted_logs}} end end @@ -405,11 +408,9 @@ defmodule Explorer.Chain.Import.Runner.Blocks do end end - defp remove_nonconsensus_logs(repo, nonconsensus_block_numbers, %{timeout: timeout}) do + defp remove_nonconsensus_logs(repo, forked_transactions_query, %{timeout: timeout}) do transaction_query = - from(transaction in Transaction, - where: transaction.block_number in ^nonconsensus_block_numbers, - select: map(transaction, [:hash]), + from(transaction in subquery(forked_transactions_query), order_by: transaction.hash ) @@ -438,7 +439,7 @@ defmodule Explorer.Chain.Import.Runner.Blocks do {:ok, deleted_logs} rescue postgrex_error in Postgrex.Error -> - {:error, %{exception: postgrex_error, block_numbers: nonconsensus_block_numbers}} + {:error, %{exception: postgrex_error, forked_transactions_query: forked_transactions_query}} end end From c038511cf2c99f244f4502d2c3491e748aaae89c Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 10 Sep 2019 13:58:00 +0300 Subject: [PATCH 23/67] use query across all inserted blocks --- .../lib/explorer/chain/import/runner/blocks.ex | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex index cbf7532ca2..5c4c42d898 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex @@ -79,7 +79,7 @@ defmodule Explorer.Chain.Import.Runner.Blocks do remove_nonconsensus_data( repo, - %{nonconsensus_block_numbers: nonconsensus_block_numbers, forked_transactions_query: where_forked}, + nonconsensus_block_numbers, insert_options ) end) @@ -363,15 +363,12 @@ defmodule Explorer.Chain.Import.Runner.Blocks do defp remove_nonconsensus_data( repo, - %{ - nonconsensus_block_numbers: nonconsensus_block_numbers, - forked_transactions_query: forked_transactions_query - }, + nonconsensus_block_numbers, insert_options ) do with {:ok, deleted_token_transfers} <- remove_nonconsensus_token_transfers(repo, nonconsensus_block_numbers, insert_options), - {:ok, deleted_logs} <- remove_nonconsensus_logs(repo, forked_transactions_query, insert_options) do + {:ok, deleted_logs} <- remove_nonconsensus_logs(repo, nonconsensus_block_numbers, insert_options) do {:ok, %{token_transfers: deleted_token_transfers, logs: deleted_logs}} end end @@ -408,9 +405,11 @@ defmodule Explorer.Chain.Import.Runner.Blocks do end end - defp remove_nonconsensus_logs(repo, forked_transactions_query, %{timeout: timeout}) do + defp remove_nonconsensus_logs(repo, nonconsensus_block_numbers, %{timeout: timeout}) do transaction_query = - from(transaction in subquery(forked_transactions_query), + from(transaction in Transaction, + where: transaction.block_number in ^nonconsensus_block_numbers, + select: map(transaction, [:hash]), order_by: transaction.hash ) @@ -439,7 +438,7 @@ defmodule Explorer.Chain.Import.Runner.Blocks do {:ok, deleted_logs} rescue postgrex_error in Postgrex.Error -> - {:error, %{exception: postgrex_error, forked_transactions_query: forked_transactions_query}} + {:error, %{exception: postgrex_error, block_numbers: nonconsensus_block_numbers}} end end From 02eb01ecb500c8fa9102e996bd675da6aca3d3bc Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 10 Sep 2019 14:31:40 +0300 Subject: [PATCH 24/67] fix try it out section --- apps/block_scout_web/assets/js/lib/try_api.js | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/apps/block_scout_web/assets/js/lib/try_api.js b/apps/block_scout_web/assets/js/lib/try_api.js index 6c33779f30..e6460f6c6b 100644 --- a/apps/block_scout_web/assets/js/lib/try_api.js +++ b/apps/block_scout_web/assets/js/lib/try_api.js @@ -55,10 +55,6 @@ function handleSuccess (query, xhr, clickedButton) { clickedButton.prop('disabled', false) } -function dropDomain (url) { - return new URL(url).pathname -} - // Show 'Try it out' UI for a module/action. $('button[data-selector*="btn-try-api"]').click(event => { const clickedButton = $(event.target) @@ -128,7 +124,7 @@ $('button[data-try-api-ui-button-type="execute"]').click(event => { } $.ajax({ - url: dropDomain(composeRequestUrl(query)), + url: composeRequestUrl(query), success: (_data, _status, xhr) => { handleSuccess(query, xhr, clickedButton) }, From 9f04a12d31fa52035b1908fc3a64c5aab7252a19 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 10 Sep 2019 14:38:59 +0300 Subject: [PATCH 25/67] fix CR issues --- apps/explorer/lib/explorer/chain/import/runner/blocks.ex | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex index 5c4c42d898..08ad720a0f 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex @@ -70,8 +70,7 @@ defmodule Explorer.Chain.Import.Runner.Blocks do lose_invalid_neighbour_consensus: lost_consensus_neighbours } -> nonconsensus_block_numbers = - lost_consensus_blocks - |> Kernel.++(lost_consensus_neighbours) + (lost_consensus_blocks ++ lost_consensus_neighbours) |> Enum.map(fn %{number: number} -> number end) @@ -378,6 +377,7 @@ defmodule Explorer.Chain.Import.Runner.Blocks do from(token_transfer in TokenTransfer, where: token_transfer.block_number in ^nonconsensus_block_numbers, select: map(token_transfer, [:transaction_hash, :log_index]), + # Enforce TokenTransfer ShareLocks order (see docs: sharelocks.md) order_by: [ token_transfer.transaction_hash, token_transfer.log_index @@ -418,11 +418,12 @@ defmodule Explorer.Chain.Import.Runner.Blocks do inner_join: transaction in subquery(transaction_query), on: log.transaction_hash == transaction.hash, select: map(log, [:transaction_hash, :index]), + # Enforce Log ShareLocks order (see docs: sharelocks.md) order_by: [ log.transaction_hash, log.index ], - lock: "FOR UPDATE" + lock: "FOR UPDATE OF l0" ) query = From ddfec17f604c3e8e752be8ddabdfe1dc391411f8 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 10 Sep 2019 14:40:09 +0300 Subject: [PATCH 26/67] add CHANGELOG entry --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 343301eb65..f942ada294 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,13 +1,14 @@ ## Current ### Features -- [#2679](https://github.com/poanetwork/blockscout/pull/2679) - added fixed height for card chain blocks and card chain transactions +- [#2679](https://github.com/poanetwork/blockscout/pull/2679) - added fixed height for card chain blocks and card chain transactions - [#2678](https://github.com/poanetwork/blockscout/pull/2678) - fixed dashboard banner height bug -- [#2672](https://github.com/poanetwork/blockscout/pull/2672) - added new theme for xUSDT +- [#2672](https://github.com/poanetwork/blockscout/pull/2672) - added new theme for xUSDT - [#2663](https://github.com/poanetwork/blockscout/pull/2663) - Fetch address counters in parallel ### Fixes - [#2682](https://github.com/poanetwork/blockscout/pull/2682) - Use Task.start instead of Task.async in caches +- [#2687](https://github.com/poanetwork/blockscout/pull/2687) - remove non-consensus token transfers, logs when inserting new consensus blocks ### Chore From 31f9a5fc1e36ed4a19192f974156cb1965d671a9 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 10 Sep 2019 14:47:04 +0300 Subject: [PATCH 27/67] add CHANGELOG entry --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 343301eb65..e0d8b60d99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,13 +1,14 @@ ## Current ### Features -- [#2679](https://github.com/poanetwork/blockscout/pull/2679) - added fixed height for card chain blocks and card chain transactions +- [#2679](https://github.com/poanetwork/blockscout/pull/2679) - added fixed height for card chain blocks and card chain transactions - [#2678](https://github.com/poanetwork/blockscout/pull/2678) - fixed dashboard banner height bug -- [#2672](https://github.com/poanetwork/blockscout/pull/2672) - added new theme for xUSDT +- [#2672](https://github.com/poanetwork/blockscout/pull/2672) - added new theme for xUSDT - [#2663](https://github.com/poanetwork/blockscout/pull/2663) - Fetch address counters in parallel ### Fixes - [#2682](https://github.com/poanetwork/blockscout/pull/2682) - Use Task.start instead of Task.async in caches +- [#2688](https://github.com/poanetwork/blockscout/pull/2688) - fix try it out section ### Chore From e569c5a7987de95b910a6393d17d12e32a6e0015 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 10 Sep 2019 20:10:06 +0300 Subject: [PATCH 28/67] add block_number index --- ...r_block_number_in_token_transfers_and_transactions.exs | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 apps/explorer/priv/repo/migrations/20190910170703_create_indexes_for_block_number_in_token_transfers_and_transactions.exs diff --git a/apps/explorer/priv/repo/migrations/20190910170703_create_indexes_for_block_number_in_token_transfers_and_transactions.exs b/apps/explorer/priv/repo/migrations/20190910170703_create_indexes_for_block_number_in_token_transfers_and_transactions.exs new file mode 100644 index 0000000000..ac15f632d5 --- /dev/null +++ b/apps/explorer/priv/repo/migrations/20190910170703_create_indexes_for_block_number_in_token_transfers_and_transactions.exs @@ -0,0 +1,8 @@ +defmodule Explorer.Repo.Migrations.CreateIndexesForBlockNumberInTokenTransfersAndTransactions do + use Ecto.Migration + + def change do + create_if_not_exists(index(:token_transfers, [:block_number])) + create_if_not_exists(index(:transactions, [:block_number])) + end +end From 71ee060f3f1d77222eea2ea21f3deb323c4c56b5 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 10 Sep 2019 20:19:55 +0300 Subject: [PATCH 29/67] remove transaction index creation --- ...exes_for_block_number_in_token_transfers_and_transactions.exs | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/explorer/priv/repo/migrations/20190910170703_create_indexes_for_block_number_in_token_transfers_and_transactions.exs b/apps/explorer/priv/repo/migrations/20190910170703_create_indexes_for_block_number_in_token_transfers_and_transactions.exs index ac15f632d5..b718224622 100644 --- a/apps/explorer/priv/repo/migrations/20190910170703_create_indexes_for_block_number_in_token_transfers_and_transactions.exs +++ b/apps/explorer/priv/repo/migrations/20190910170703_create_indexes_for_block_number_in_token_transfers_and_transactions.exs @@ -3,6 +3,5 @@ defmodule Explorer.Repo.Migrations.CreateIndexesForBlockNumberInTokenTransfersAn def change do create_if_not_exists(index(:token_transfers, [:block_number])) - create_if_not_exists(index(:transactions, [:block_number])) end end From a883e4ac68f351a5f7c98d4300b89326dc6dacd2 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Wed, 11 Sep 2019 15:20:16 +0300 Subject: [PATCH 30/67] deduplicate numbers --- apps/explorer/lib/explorer/chain/import/runner/blocks.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex index 08ad720a0f..aa80e6c251 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex @@ -75,6 +75,7 @@ defmodule Explorer.Chain.Import.Runner.Blocks do number end) |> Enum.sort() + |> Enum.uniq() remove_nonconsensus_data( repo, From b674fa675e943ed888aa212b10a6ddd859922cdd Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Wed, 11 Sep 2019 16:08:23 +0300 Subject: [PATCH 31/67] Update blocks.ex --- apps/explorer/lib/explorer/chain/import/runner/blocks.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex index aa80e6c251..825a533743 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex @@ -75,7 +75,7 @@ defmodule Explorer.Chain.Import.Runner.Blocks do number end) |> Enum.sort() - |> Enum.uniq() + |> Enum.dedup() remove_nonconsensus_data( repo, From fe7115b0e270b5018028ccaf513630dc448b518d Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Wed, 11 Sep 2019 16:11:10 +0300 Subject: [PATCH 32/67] fix exchange rate websocket update for Rootstock Rootstock has custom logic for market cap calculation that uses data from DB. This PR add required feilds to exchange_rate when sending it through a web socket. --- apps/block_scout_web/lib/block_scout_web/notifier.ex | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/apps/block_scout_web/lib/block_scout_web/notifier.ex b/apps/block_scout_web/lib/block_scout_web/notifier.ex index aa47163c98..03e46cf38b 100644 --- a/apps/block_scout_web/lib/block_scout_web/notifier.ex +++ b/apps/block_scout_web/lib/block_scout_web/notifier.ex @@ -7,6 +7,7 @@ defmodule BlockScoutWeb.Notifier do alias BlockScoutWeb.{AddressContractVerificationView, Endpoint} alias Explorer.{Chain, Market, Repo} alias Explorer.Chain.{Address, InternalTransaction, Transaction} + alias Explorer.Chain.Supply.RSK alias Explorer.Counters.AverageBlockTime alias Explorer.ExchangeRates.Token alias Explorer.SmartContract.{Solidity.CodeCompiler, Solidity.CompilerVersion} @@ -76,8 +77,17 @@ defmodule BlockScoutWeb.Notifier do data -> data end + exchange_rate_with_available_supply = + case Application.get_env(:explorer, :supply) do + RSK -> + %{exchange_rate | available_supply: RSK.circulating(), market_cap_usd: RSK.market_cap(exchange_rate)} + + _ -> + exchange_rate + end + Endpoint.broadcast("exchange_rate:new_rate", "new_rate", %{ - exchange_rate: exchange_rate, + exchange_rate: exchange_rate_with_available_supply, market_history_data: Enum.map(market_history_data, fn day -> Map.take(day, [:closing_price, :date]) end) }) end From df7c043d6a9c45959e2d83358b764e41bf9a7b7b Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Wed, 11 Sep 2019 16:14:51 +0300 Subject: [PATCH 33/67] add CHANGELOG entry --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c85224762..9084f2de1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ### Features - [#2665](https://github.com/poanetwork/blockscout/pull/2665) - new menu layout for mobile devices -- [#2679](https://github.com/poanetwork/blockscout/pull/2679) - added fixed height for card chain blocks and card chain transactions +- [#2679](https://github.com/poanetwork/blockscout/pull/2679) - added fixed height for card chain blocks and card chain transactions - [#2678](https://github.com/poanetwork/blockscout/pull/2678) - fixed dashboard banner height bug - [#2672](https://github.com/poanetwork/blockscout/pull/2672) - added new theme for xUSDT - [#2663](https://github.com/poanetwork/blockscout/pull/2663) - Fetch address counters in parallel @@ -10,7 +10,8 @@ ### Fixes - [#2684](https://github.com/poanetwork/blockscout/pull/2684) - do not filter pending logs - [#2682](https://github.com/poanetwork/blockscout/pull/2682) - Use Task.start instead of Task.async in caches -- [#2671](https://github.com/poanetwork/blockscout/pull/2671) - fixed buttons color at smart contract section +- [#2671](https://github.com/poanetwork/blockscout/pull/2671) - fixed buttons color at smart contract section +- [#2691](https://github.com/poanetwork/blockscout/pull/2691) - fix exchange rate websocket update for Rootstock ### Chore From 7c660b472ba894bf48fdf5460d75dc547ccde566 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Thu, 12 Sep 2019 12:49:18 +0300 Subject: [PATCH 34/67] do not update chart --- apps/block_scout_web/lib/block_scout_web/notifier.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/block_scout_web/lib/block_scout_web/notifier.ex b/apps/block_scout_web/lib/block_scout_web/notifier.ex index 03e46cf38b..7c07c8524e 100644 --- a/apps/block_scout_web/lib/block_scout_web/notifier.ex +++ b/apps/block_scout_web/lib/block_scout_web/notifier.ex @@ -80,7 +80,7 @@ defmodule BlockScoutWeb.Notifier do exchange_rate_with_available_supply = case Application.get_env(:explorer, :supply) do RSK -> - %{exchange_rate | available_supply: RSK.circulating(), market_cap_usd: RSK.market_cap(exchange_rate)} + %{exchange_rate | available_supply: nil, market_cap_usd: RSK.market_cap(exchange_rate)} _ -> exchange_rate From 9e4b2344120f1f184a855939654057b1c71a06e8 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Thu, 12 Sep 2019 15:31:45 +0300 Subject: [PATCH 35/67] remove nonconsensus internal transactions --- .../explorer/chain/import/runner/blocks.ex | 51 ++++++++++++++++++- .../chain/import/runner/blocks_test.exs | 26 +++++++++- 2 files changed, 74 insertions(+), 3 deletions(-) diff --git a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex index 825a533743..9cfb771736 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex @@ -368,8 +368,15 @@ defmodule Explorer.Chain.Import.Runner.Blocks do ) do with {:ok, deleted_token_transfers} <- remove_nonconsensus_token_transfers(repo, nonconsensus_block_numbers, insert_options), - {:ok, deleted_logs} <- remove_nonconsensus_logs(repo, nonconsensus_block_numbers, insert_options) do - {:ok, %{token_transfers: deleted_token_transfers, logs: deleted_logs}} + {:ok, deleted_logs} <- remove_nonconsensus_logs(repo, nonconsensus_block_numbers, insert_options), + {:ok, deleted_internal_transactions} <- + remove_nonconsensus_internal_transactions(repo, nonconsensus_block_numbers, insert_options) do + {:ok, + %{ + token_transfers: deleted_token_transfers, + logs: deleted_logs, + internal_transactions: deleted_internal_transactions + }} end end @@ -406,6 +413,46 @@ defmodule Explorer.Chain.Import.Runner.Blocks do end end + defp remove_nonconsensus_internal_transactions(repo, nonconsensus_block_numbers, %{timeout: timeout}) do + transaction_query = + from(transaction in Transaction, + where: transaction.block_number in ^nonconsensus_block_numbers, + select: map(transaction, [:hash]), + order_by: transaction.hash + ) + + ordered_internal_transactions = + from(internal_transaction in InternalTransaction, + inner_join: transaction in subquery(transaction_query), + on: internal_transaction.transaction_hash == transaction.hash, + select: map(internal_transaction, [:transaction_hash, :index]), + # Enforce Log ShareLocks order (see docs: sharelocks.md) + order_by: [ + internal_transaction.transaction_hash, + internal_transaction.index + ], + lock: "FOR UPDATE OF i0" + ) + + query = + from(internal_transaction in InternalTransaction, + select: map(internal_transaction, [:transaction_hash, :index]), + inner_join: ordered_internal_transaction in subquery(ordered_internal_transactions), + on: + ordered_internal_transaction.transaction_hash == internal_transaction.transaction_hash and + ordered_internal_transaction.index == internal_transaction.index + ) + + try do + {_count, deleted_internal_transactions} = repo.delete_all(query, timeout: timeout) + + {:ok, deleted_internal_transactions} + rescue + postgrex_error in Postgrex.Error -> + {:error, %{exception: postgrex_error, block_numbers: nonconsensus_block_numbers}} + end + end + defp remove_nonconsensus_logs(repo, nonconsensus_block_numbers, %{timeout: timeout}) do transaction_query = from(transaction in Transaction, diff --git a/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs b/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs index 459363839b..96104a0528 100644 --- a/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs +++ b/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs @@ -7,7 +7,7 @@ defmodule Explorer.Chain.Import.Runner.BlocksTest do alias Ecto.Multi alias Explorer.Chain.Import.Runner.{Blocks, Transactions} - alias Explorer.Chain.{Address, Block, Log, Transaction, TokenTransfer} + alias Explorer.Chain.{Address, Block, InternalTransaction, Log, Transaction, TokenTransfer} alias Explorer.Chain alias Explorer.Repo @@ -177,6 +177,30 @@ defmodule Explorer.Chain.Import.Runner.BlocksTest do assert count(Log) == 0 end + test "remove_nonconsensus_date deletes nonconsensus internal transactions", %{ + consensus_block: %{number: block_number} = block, + options: options + } do + old_block = insert(:block, number: block_number, consensus: true) + forked_transaction = :transaction |> insert() |> with_block(old_block) + + %InternalTransaction{index: index, transaction_hash: hash} = + insert(:internal_transaction, index: 0, transaction: forked_transaction) + + assert count(InternalTransaction) == 1 + + assert {:ok, + %{ + remove_nonconsensus_data: %{ + internal_transactions: [ + %{transaction_hash: ^hash, index: ^index} + ] + } + }} = run_block_consensus_change(block, true, options) + + assert count(InternalTransaction) == 0 + end + test "derive_address_current_token_balances inserts rows if there is an address_token_balance left for the rows deleted by delete_address_current_token_balances", %{consensus_block: %{number: block_number} = block, options: options} do token = insert(:token) From d7c62b2a5ae7fa6373e243a3b0c3f79661ac3abc Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Thu, 12 Sep 2019 15:33:21 +0300 Subject: [PATCH 36/67] remove update of block number for internal transactions --- .../explorer/chain/import/runner/blocks.ex | 25 ------------------- 1 file changed, 25 deletions(-) diff --git a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex index 9cfb771736..5a69e1778a 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex @@ -131,31 +131,6 @@ defmodule Explorer.Chain.Import.Runner.Blocks do } ) end) - |> Multi.run( - :internal_transaction_transaction_block_number, - fn repo, %{blocks: blocks} -> - blocks_hashes = Enum.map(blocks, & &1.hash) - - query = - from( - internal_transaction in InternalTransaction, - join: transaction in Transaction, - on: internal_transaction.transaction_hash == transaction.hash, - join: block in Block, - on: block.hash == transaction.block_hash, - where: block.hash in ^blocks_hashes, - update: [ - set: [ - block_number: block.number - ] - ] - ) - - {total, _} = repo.update_all(query, []) - - {:ok, total} - end - ) end @impl Runner From fdecb070623211bfcfa6f7194e8c6fc7ead03fa9 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Thu, 12 Sep 2019 15:36:47 +0300 Subject: [PATCH 37/67] add CHANGELOG entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e0fd3e1b9..bc9ed39546 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - [#2663](https://github.com/poanetwork/blockscout/pull/2663) - Fetch address counters in parallel ### Fixes +- [#2693](https://github.com/poanetwork/blockscout/pull/2693) - remove non consensus internal transactions - [#2687](https://github.com/poanetwork/blockscout/pull/2687) - remove non-consensus token transfers, logs when inserting new consensus blocks - [#2684](https://github.com/poanetwork/blockscout/pull/2684) - do not filter pending logs - [#2682](https://github.com/poanetwork/blockscout/pull/2682) - Use Task.start instead of Task.async in caches From 26b05b184f2484ca4e2d978a90f58e00fa4c79de Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Thu, 12 Sep 2019 15:50:29 +0300 Subject: [PATCH 38/67] fix typo --- .../explorer/chain/import/runner/blocks.ex | 25 +++++++++++++++++++ .../chain/import/runner/blocks_test.exs | 2 +- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex index 5a69e1778a..9cfb771736 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex @@ -131,6 +131,31 @@ defmodule Explorer.Chain.Import.Runner.Blocks do } ) end) + |> Multi.run( + :internal_transaction_transaction_block_number, + fn repo, %{blocks: blocks} -> + blocks_hashes = Enum.map(blocks, & &1.hash) + + query = + from( + internal_transaction in InternalTransaction, + join: transaction in Transaction, + on: internal_transaction.transaction_hash == transaction.hash, + join: block in Block, + on: block.hash == transaction.block_hash, + where: block.hash in ^blocks_hashes, + update: [ + set: [ + block_number: block.number + ] + ] + ) + + {total, _} = repo.update_all(query, []) + + {:ok, total} + end + ) end @impl Runner diff --git a/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs b/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs index 96104a0528..5a5d547068 100644 --- a/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs +++ b/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs @@ -177,7 +177,7 @@ defmodule Explorer.Chain.Import.Runner.BlocksTest do assert count(Log) == 0 end - test "remove_nonconsensus_date deletes nonconsensus internal transactions", %{ + test "remove_nonconsensus_data deletes nonconsensus internal transactions", %{ consensus_block: %{number: block_number} = block, options: options } do From 0952b2a8288fa827b7454174c0ac8deaf567f059 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Fri, 13 Sep 2019 11:47:25 +0300 Subject: [PATCH 39/67] do not update fetched_coin_balance with nil Currently, we update `fetched_coin_balance` field of an address to nil if changes have `fetched_coin_balance_block_number` bigger than current value in the DB. --- .../explorer/chain/import/runner/addresses.ex | 6 +- .../chain/import/runner/addresses_test.exs | 69 +++++++++++++++++++ .../chain/import/runner/blocks_test.exs | 5 +- 3 files changed, 75 insertions(+), 5 deletions(-) create mode 100644 apps/explorer/test/explorer/chain/import/runner/addresses_test.exs diff --git a/apps/explorer/lib/explorer/chain/import/runner/addresses.ex b/apps/explorer/lib/explorer/chain/import/runner/addresses.ex index 564c36452b..e277592854 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/addresses.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/addresses.ex @@ -104,13 +104,15 @@ defmodule Explorer.Chain.Import.Runner.Addresses do fetched_coin_balance: fragment( """ - CASE WHEN EXCLUDED.fetched_coin_balance_block_number IS NOT NULL AND - (? IS NULL OR + CASE WHEN EXCLUDED.fetched_coin_balance_block_number IS NOT NULL + AND EXCLUDED.fetched_coin_balance IS NOT NULL AND + (? IS NULL OR ? IS NULL OR EXCLUDED.fetched_coin_balance_block_number >= ?) THEN EXCLUDED.fetched_coin_balance ELSE ? END """, + address.fetched_coin_balance, address.fetched_coin_balance_block_number, address.fetched_coin_balance_block_number, address.fetched_coin_balance diff --git a/apps/explorer/test/explorer/chain/import/runner/addresses_test.exs b/apps/explorer/test/explorer/chain/import/runner/addresses_test.exs new file mode 100644 index 0000000000..7ef8292ee8 --- /dev/null +++ b/apps/explorer/test/explorer/chain/import/runner/addresses_test.exs @@ -0,0 +1,69 @@ +defmodule Explorer.Chain.Import.Runner.AddressesTest do + use Explorer.DataCase + + alias Ecto.Multi + alias Explorer.Chain.{Address, Wei} + alias Explorer.Chain.Import.Runner.Addresses + alias Explorer.Repo + + describe "run/1" do + test "does not update fetched_coin_balance if original value is not nil but new value is nil" do + block_number = 5 + original_address = insert(:address, fetched_coin_balance: 5, fetched_coin_balance_block_number: block_number) + + new_params = %{ + fetched_coin_balance: nil, + fetched_coin_balance_block_number: block_number, + hash: to_string(original_address.hash) + } + + changeset = Address.balance_changeset(%Address{}, new_params) + + wei = original_address.fetched_coin_balance + + assert {:ok, + %{ + addresses: [ + %Address{ + fetched_coin_balance: ^wei, + fetched_coin_balance_block_number: 5 + } + ] + }} = run([changeset.changes]) + end + + test "updates fetched_coin_balance if original value is nil and new value is not nil" do + block_number = 5 + original_address = insert(:address, fetched_coin_balance: nil, fetched_coin_balance_block_number: block_number) + + new_params = %{ + fetched_coin_balance: 5, + fetched_coin_balance_block_number: block_number, + hash: to_string(original_address.hash) + } + + changeset = Address.balance_changeset(%Address{}, new_params) + + wei = %Wei{value: Decimal.new(new_params.fetched_coin_balance)} + + assert {:ok, + %{ + addresses: [ + %Address{ + fetched_coin_balance: ^wei, + fetched_coin_balance_block_number: 5 + } + ] + }} = run([changeset.changes]) + end + end + + defp run(changes) do + timestamp = DateTime.utc_now() + options = %{timestamps: %{inserted_at: timestamp, updated_at: timestamp}} + + Multi.new() + |> Addresses.run(changes, options) + |> Repo.transaction() + end +end diff --git a/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs b/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs index 459363839b..4b1ebf4ef7 100644 --- a/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs +++ b/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs @@ -7,9 +7,8 @@ defmodule Explorer.Chain.Import.Runner.BlocksTest do alias Ecto.Multi alias Explorer.Chain.Import.Runner.{Blocks, Transactions} - alias Explorer.Chain.{Address, Block, Log, Transaction, TokenTransfer} - alias Explorer.Chain - alias Explorer.Repo + alias Explorer.Chain.{Address, Block, Log, TokenTransfer, Transaction} + alias Explorer.{Chain, Repo} describe "run/1" do setup do From d06c1476f5abfb1d2a9b892caa38389f41e29c82 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Fri, 13 Sep 2019 11:54:11 +0300 Subject: [PATCH 40/67] add CHANGELOG entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e0fd3e1b9..ce56ced859 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - [#2663](https://github.com/poanetwork/blockscout/pull/2663) - Fetch address counters in parallel ### Fixes +- [#2696](https://github.com/poanetwork/blockscout/pull/2696) - do not update fetched_coin_balance with nil - [#2687](https://github.com/poanetwork/blockscout/pull/2687) - remove non-consensus token transfers, logs when inserting new consensus blocks - [#2684](https://github.com/poanetwork/blockscout/pull/2684) - do not filter pending logs - [#2682](https://github.com/poanetwork/blockscout/pull/2682) - Use Task.start instead of Task.async in caches From 86d9786f55701562d670c3a58205c8367d4200e0 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Mon, 16 Sep 2019 13:39:11 +0300 Subject: [PATCH 41/67] Update blocks.ex --- apps/explorer/lib/explorer/chain/import/runner/blocks.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex index 9cfb771736..1c58446292 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex @@ -426,7 +426,7 @@ defmodule Explorer.Chain.Import.Runner.Blocks do inner_join: transaction in subquery(transaction_query), on: internal_transaction.transaction_hash == transaction.hash, select: map(internal_transaction, [:transaction_hash, :index]), - # Enforce Log ShareLocks order (see docs: sharelocks.md) + # Enforce InternalTransaction ShareLocks order (see docs: sharelocks.md) order_by: [ internal_transaction.transaction_hash, internal_transaction.index From 5d9f7d0c20528437e8976d55ad3dd7e804bd41ef Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Mon, 16 Sep 2019 17:53:35 +0300 Subject: [PATCH 42/67] Exclude nonconsensus blocks from avg block time calculation by default --- CHANGELOG.md | 1 + apps/explorer/config/config.exs | 2 +- docs/env-variables.md | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e0fd3e1b9..72877501b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - [#2663](https://github.com/poanetwork/blockscout/pull/2663) - Fetch address counters in parallel ### Fixes +- [#2701](https://github.com/poanetwork/blockscout/pull/2701) - Exclude nonconsensus blocks from avg block time calculation by default - [#2687](https://github.com/poanetwork/blockscout/pull/2687) - remove non-consensus token transfers, logs when inserting new consensus blocks - [#2684](https://github.com/poanetwork/blockscout/pull/2684) - do not filter pending logs - [#2682](https://github.com/poanetwork/blockscout/pull/2682) - Use Task.start instead of Task.async in caches diff --git a/apps/explorer/config/config.exs b/apps/explorer/config/config.exs index deefc9e3da..8a264eaf33 100644 --- a/apps/explorer/config/config.exs +++ b/apps/explorer/config/config.exs @@ -14,7 +14,7 @@ config :explorer, System.get_env("ALLOWED_EVM_VERSIONS") || "homestead,tangerineWhistle,spuriousDragon,byzantium,constantinople,petersburg,default", include_uncles_in_average_block_time: - if(System.get_env("UNCLES_IN_AVERAGE_BLOCK_TIME") == "false", do: false, else: true), + if(System.get_env("UNCLES_IN_AVERAGE_BLOCK_TIME") == "true", do: true, else: false), healthy_blocks_period: System.get_env("HEALTHY_BLOCKS_PERIOD") || :timer.minutes(5) average_block_period = diff --git a/docs/env-variables.md b/docs/env-variables.md index 5eef49d07d..503689d091 100644 --- a/docs/env-variables.md +++ b/docs/env-variables.md @@ -56,6 +56,7 @@ $ export NETWORK=POA | `SUPPORTED_CHAINS` | | Array of supported chains that displays in the footer and in the chains dropdown. This var was introduced in this PR [#1900](https://github.com/poanetwork/blockscout/pull/1900) and looks like an array of JSON objects. | (empty) | v2.0.0+ | | | | `BLOCK_COUNT_CACHE_PERIOD ` | | time to live of cache in seconds. This var was introduced in [#1876](https://github.com/poanetwork/blockscout/pull/1876) | 600 | v2.0.0+ | | | | `ALLOWED_EVM_VERSIONS ` | | the comma-separated list of allowed EVM versions for contracts verification. This var was introduced in [#1964](https://github.com/poanetwork/blockscout/pull/1964) | "homestead, tangerineWhistle, spuriousDragon, byzantium, constantinople, petersburg" | v2.0.0+ | | | +| `UNCLES_IN_AVERAGE_BLOCK_TIME` | Include or exclude nonconsensus blocks in avg block time calculation. Exclude if `false`. | false | v2.0.1+ | | | | `AVERAGE_BLOCK_CACHE_PERIOD` | | Update of average block cache, in seconds | 30 minutes | v2.0.2+ | | | `MARKET_HISTORY_CACHE_PERIOD` | | Update of market history cache, in seconds | 6 hours | v2.0.2+ | | | `DISABLE_WEBAPP` | | If `true`, endpoints to webapp are hidden (compile-time) | `false` | v2.0.3+ | :white_check_mark: | | From a8cd5058cf72897264981072bd3ee5fe316e884c Mon Sep 17 00:00:00 2001 From: pasqu4le Date: Fri, 23 Aug 2019 18:00:01 +0200 Subject: [PATCH 43/67] Enforce DB transaction's order to prevent deadlocks Problem: due to the ShareLock mechanism used by PostgreSQL when there are several DB transactions are acting on multiple rows of the same table, it's possible to incur in a deadlock and so into an error. Solution: enforce a consistent order of ShareLock acquisition in all the relevant transaction. Also add clear documentation that should help in the future. --- CHANGELOG.md | 1 + apps/explorer/lib/explorer/chain.ex | 36 +- .../lib/explorer/chain/contract_method.ex | 5 +- .../import/runner/address/coin_balances.ex | 2 +- .../runner/address/current_token_balances.ex | 12 +- .../import/runner/address/token_balances.ex | 2 +- .../explorer/chain/import/runner/addresses.ex | 15 +- .../chain/import/runner/block/rewards.ex | 2 +- .../runner/block/second_degree_relations.ex | 2 +- .../explorer/chain/import/runner/blocks.ex | 342 ++++++++++-------- .../import/runner/internal_transactions.ex | 22 +- ...internal_transactions_indexed_at_blocks.ex | 30 +- .../lib/explorer/chain/import/runner/logs.ex | 2 +- .../chain/import/runner/staking_pools.ex | 33 +- .../import/runner/staking_pools_delegators.ex | 5 +- .../chain/import/runner/token_transfers.ex | 2 +- .../explorer/chain/import/runner/tokens.ex | 123 +++---- .../chain/import/runner/transaction/forks.ex | 4 +- .../chain/import/runner/transactions.ex | 28 +- .../explorer/chain_spec/parity/importer.ex | 25 +- .../lib/explorer/chain_spec/poa/importer.ex | 25 +- apps/explorer/lib/explorer/market/market.ex | 5 +- .../explorer/validator/metadata_importer.ex | 5 +- .../temporary/blocks_transactions_mismatch.ex | 31 +- docs/sharelocks.md | 88 +++++ 25 files changed, 535 insertions(+), 312 deletions(-) create mode 100644 docs/sharelocks.md diff --git a/CHANGELOG.md b/CHANGELOG.md index bc9ed39546..a403e6b61c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ - [#2497](https://github.com/poanetwork/blockscout/pull/2497) - Add generic Ordered Cache behaviour and implementation ### Fixes +- [#2619](https://github.com/poanetwork/blockscout/pull/2619) - Enforce DB transaction's order to prevent deadlocks - [#2659](https://github.com/poanetwork/blockscout/pull/2659) - Multipurpose front-end part update - [#2640](https://github.com/poanetwork/blockscout/pull/2640) - SVG network icons - [#2635](https://github.com/poanetwork/blockscout/pull/2635) - optimize ERC721 inventory query diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index 15f6644f25..9b70373228 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -8,6 +8,7 @@ defmodule Explorer.Chain do from: 2, join: 4, limit: 2, + lock: 2, order_by: 2, order_by: 3, offset: 2, @@ -2924,14 +2925,20 @@ defmodule Explorer.Chain do ]) :: {integer(), nil | [term()]} def find_and_update_replaced_transactions(transactions, timeout \\ :infinity) do query = - Enum.reduce(transactions, Transaction, fn %{hash: hash, nonce: nonce, from_address_hash: from_address_hash}, - query -> - from(t in query, - or_where: - t.nonce == ^nonce and t.from_address_hash == ^from_address_hash and t.hash != ^hash and - not is_nil(t.block_number) - ) - end) + transactions + |> Enum.reduce( + Transaction, + fn %{hash: hash, nonce: nonce, from_address_hash: from_address_hash}, query -> + from(t in query, + or_where: + t.nonce == ^nonce and t.from_address_hash == ^from_address_hash and t.hash != ^hash and + not is_nil(t.block_number) + ) + end + ) + # Enforce Transaction ShareLocks order (see docs: sharelocks.md) + |> order_by(asc: :hash) + |> lock("FOR UPDATE") hashes = Enum.map(transactions, & &1.hash) @@ -2974,10 +2981,15 @@ defmodule Explorer.Chain do or_where: t.nonce == ^nonce and t.from_address_hash == ^from_address and is_nil(t.block_hash) ) end) - - update_query = from(t in query, update: [set: [status: ^:error, error: "dropped/replaced"]]) - - Repo.update_all(update_query, [], timeout: timeout) + # Enforce Transaction ShareLocks order (see docs: sharelocks.md) + |> order_by(asc: :hash) + |> lock("FOR UPDATE") + + Repo.update_all( + from(t in Transaction, join: s in subquery(query), on: t.hash == s.hash), + [set: [error: "dropped/replaced", status: :error]], + timeout: timeout + ) end end diff --git a/apps/explorer/lib/explorer/chain/contract_method.ex b/apps/explorer/lib/explorer/chain/contract_method.ex index 5e2caa99c5..ed24f395ee 100644 --- a/apps/explorer/lib/explorer/chain/contract_method.ex +++ b/apps/explorer/lib/explorer/chain/contract_method.ex @@ -46,7 +46,10 @@ defmodule Explorer.Chain.ContractMethod do end) end - Repo.insert_all(__MODULE__, successes, on_conflict: :nothing, conflict_target: [:identifier, :abi]) + # Enforce ContractMethod ShareLocks order (see docs: sharelocks.md) + ordered_successes = Enum.sort_by(successes, &{&1.identifier, &1.abi}) + + Repo.insert_all(__MODULE__, ordered_successes, on_conflict: :nothing, conflict_target: [:identifier, :abi]) end def import_all do diff --git a/apps/explorer/lib/explorer/chain/import/runner/address/coin_balances.ex b/apps/explorer/lib/explorer/chain/import/runner/address/coin_balances.ex index 16c3f611bc..ecc67cd9d4 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/address/coin_balances.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/address/coin_balances.ex @@ -71,7 +71,7 @@ defmodule Explorer.Chain.Import.Runner.Address.CoinBalances do defp insert(repo, changes_list, %{timeout: timeout, timestamps: timestamps} = options) when is_list(changes_list) do on_conflict = Map.get_lazy(options, :on_conflict, &default_on_conflict/0) - # order so that row ShareLocks are grabbed in a consistent order + # Enforce CoinBalance ShareLocks order (see docs: sharelocks.md) ordered_changes_list = Enum.sort_by(changes_list, &{&1.address_hash, &1.block_number}) {:ok, _} = diff --git a/apps/explorer/lib/explorer/chain/import/runner/address/current_token_balances.ex b/apps/explorer/lib/explorer/chain/import/runner/address/current_token_balances.ex index d6259baad8..503d5c877b 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/address/current_token_balances.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/address/current_token_balances.ex @@ -106,12 +106,9 @@ defmodule Explorer.Chain.Import.Runner.Address.CurrentTokenBalances do |> Map.put_new(:timeout, @timeout) |> Map.put(:timestamps, timestamps) - # order so that row ShareLocks are grabbed in a consistent order - ordered_changes_list = Enum.sort_by(changes_list, &{&1.address_hash, &1.token_contract_address_hash}) - multi |> Multi.run(:address_current_token_balances, fn repo, _ -> - insert(repo, ordered_changes_list, insert_options) + insert(repo, changes_list, insert_options) end) |> Multi.run(:address_current_token_balances_update_token_holder_counts, fn repo, %{ @@ -193,10 +190,13 @@ defmodule Explorer.Chain.Import.Runner.Address.CurrentTokenBalances do }) :: {:ok, [CurrentTokenBalance.t()]} | {:error, [Changeset.t()]} - defp insert(repo, ordered_changes_list, %{timeout: timeout, timestamps: timestamps} = options) - when is_atom(repo) and is_list(ordered_changes_list) do + defp insert(repo, changes_list, %{timeout: timeout, timestamps: timestamps} = options) + when is_atom(repo) and is_list(changes_list) do on_conflict = Map.get_lazy(options, :on_conflict, &default_on_conflict/0) + # Enforce CurrentTokenBalance ShareLocks order (see docs: sharelocks.md) + ordered_changes_list = Enum.sort_by(changes_list, &{&1.address_hash, &1.token_contract_address_hash}) + Import.insert_changes_list( repo, ordered_changes_list, diff --git a/apps/explorer/lib/explorer/chain/import/runner/address/token_balances.ex b/apps/explorer/lib/explorer/chain/import/runner/address/token_balances.ex index 616a714583..ff2464eb21 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/address/token_balances.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/address/token_balances.ex @@ -59,7 +59,7 @@ defmodule Explorer.Chain.Import.Runner.Address.TokenBalances do def insert(repo, changes_list, %{timeout: timeout, timestamps: timestamps} = options) when is_list(changes_list) do on_conflict = Map.get_lazy(options, :on_conflict, &default_on_conflict/0) - # order so that row ShareLocks are grabbed in a consistent order + # Enforce TokenBalance ShareLocks order (see docs: sharelocks.md) ordered_changes_list = Enum.sort_by(changes_list, &{&1.address_hash, &1.token_contract_address_hash, &1.block_number}) diff --git a/apps/explorer/lib/explorer/chain/import/runner/addresses.ex b/apps/explorer/lib/explorer/chain/import/runner/addresses.ex index 564c36452b..74747a7d2a 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/addresses.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/addresses.ex @@ -80,7 +80,7 @@ defmodule Explorer.Chain.Import.Runner.Addresses do defp insert(repo, changes_list, %{timeout: timeout, timestamps: timestamps} = options) when is_list(changes_list) do on_conflict = Map.get_lazy(options, :on_conflict, &default_on_conflict/0) - # order so that row ShareLocks are grabbed in a consistent order + # Enforce Address ShareLocks order (see docs: sharelocks.md) ordered_changes_list = sort_changes_list(changes_list) Import.insert_changes_list( @@ -153,13 +153,18 @@ defmodule Explorer.Chain.Import.Runner.Addresses do query = from(t in Transaction, where: t.created_contract_address_hash in ^ordered_created_contract_hashes, - update: [ - set: [created_contract_code_indexed_at: ^timestamps.updated_at] - ] + # Enforce Transaction ShareLocks order (see docs: sharelocks.md) + order_by: t.hash, + lock: "FOR UPDATE" ) try do - {_, result} = repo.update_all(query, [], timeout: timeout) + {_, result} = + repo.update_all( + from(t in Transaction, join: s in subquery(query), on: t.hash == s.hash), + [set: [created_contract_code_indexed_at: timestamps.updated_at]], + timeout: timeout + ) {:ok, result} rescue diff --git a/apps/explorer/lib/explorer/chain/import/runner/block/rewards.ex b/apps/explorer/lib/explorer/chain/import/runner/block/rewards.ex index 3ce55fbf07..4746a297fd 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/block/rewards.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/block/rewards.ex @@ -52,7 +52,7 @@ defmodule Explorer.Chain.Import.Runner.Block.Rewards do when is_list(changes_list) do on_conflict = Map.get_lazy(options, :on_conflict, &default_on_conflict/0) - # order so that row ShareLocks are grabbed in a consistent order + # Enforce Reward ShareLocks order (see docs: sharelocks.md) ordered_changes_list = Enum.sort_by(changes_list, &{&1.address_hash, &1.address_type, &1.block_hash}) Import.insert_changes_list( diff --git a/apps/explorer/lib/explorer/chain/import/runner/block/second_degree_relations.ex b/apps/explorer/lib/explorer/chain/import/runner/block/second_degree_relations.ex index 8c8f4e26c4..18b8eb089c 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/block/second_degree_relations.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/block/second_degree_relations.ex @@ -62,7 +62,7 @@ defmodule Explorer.Chain.Import.Runner.Block.SecondDegreeRelations do defp insert(repo, changes_list, %{timeout: timeout} = options) when is_atom(repo) and is_list(changes_list) do on_conflict = Map.get_lazy(options, :on_conflict, &default_on_conflict/0) - # order so that row ShareLocks are grabbed in a consistent order + # Enforce SeconDegreeRelation ShareLocks order (see docs: sharelocks.md) ordered_changes_list = Enum.sort_by(changes_list, &{&1.nephew_hash, &1.uncle_hash}) Import.insert_changes_list(repo, ordered_changes_list, diff --git a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex index 1c58446292..58cfb67ce5 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex @@ -5,9 +5,8 @@ defmodule Explorer.Chain.Import.Runner.Blocks do require Ecto.Query - import Ecto.Query, only: [from: 2, select: 2, subquery: 1, update: 2] + import Ecto.Query, only: [from: 2, lock: 2, order_by: 2, subquery: 1] - alias Ecto.Adapters.SQL alias Ecto.{Changeset, Multi, Repo} alias Explorer.Chain.{Address, Block, Hash, Import, InternalTransaction, Log, TokenTransfer, Transaction} alias Explorer.Chain.Block.Reward @@ -131,76 +130,60 @@ defmodule Explorer.Chain.Import.Runner.Blocks do } ) end) - |> Multi.run( - :internal_transaction_transaction_block_number, - fn repo, %{blocks: blocks} -> - blocks_hashes = Enum.map(blocks, & &1.hash) - - query = - from( - internal_transaction in InternalTransaction, - join: transaction in Transaction, - on: internal_transaction.transaction_hash == transaction.hash, - join: block in Block, - on: block.hash == transaction.block_hash, - where: block.hash in ^blocks_hashes, - update: [ - set: [ - block_number: block.number - ] - ] - ) - - {total, _} = repo.update_all(query, []) - - {:ok, total} - end - ) + |> Multi.run(:internal_transaction_transaction_block_number, fn repo, %{blocks: blocks} when is_list(blocks) -> + update_internal_transaction_block_number(repo, blocks) + end) end @impl Runner def timeout, do: @timeout - # sobelow_skip ["SQL.Query"] defp derive_transaction_forks(%{ repo: repo, timeout: timeout, timestamps: %{inserted_at: inserted_at, updated_at: updated_at}, where_forked: where_forked }) do - query = - from(transaction in where_forked, - select: [ - transaction.block_hash, - transaction.index, - transaction.hash, - type(^inserted_at, transaction.inserted_at), - type(^updated_at, transaction.updated_at) - ], - # order so that row ShareLocks are grabbed in a consistent order with - # `Explorer.Chain.Import.Runner.Transactions.insert` - order_by: transaction.hash - ) + multi = + Multi.new() + |> Multi.run(:get_forks, fn repo, _ -> + query = + from(transaction in where_forked, + select: %{ + uncle_hash: transaction.block_hash, + index: transaction.index, + hash: transaction.hash, + inserted_at: type(^inserted_at, transaction.inserted_at), + updated_at: type(^updated_at, transaction.updated_at) + } + ) + + transactions = repo.all(query) + {:ok, transactions} + end) + |> Multi.run(:insert_transaction_forks, fn repo, %{get_forks: transactions} -> + # Enforce Fork ShareLocks order (see docs: sharelocks.md) + ordered_forks = Enum.sort_by(transactions, &{&1.uncle_hash, &1.index}) + + {_total, result} = + repo.insert_all( + Transaction.Fork, + ordered_forks, + conflict_target: [:uncle_hash, :index], + on_conflict: + from( + transaction_fork in Transaction.Fork, + update: [set: [hash: fragment("EXCLUDED.hash")]], + where: fragment("EXCLUDED.hash <> ?", transaction_fork.hash) + ), + returning: [:uncle_hash, :hash] + ) + + {:ok, result} + end) - {select_sql, parameters} = SQL.to_sql(:all, repo, query) - - insert_sql = """ - INSERT INTO transaction_forks (uncle_hash, index, hash, inserted_at, updated_at) - #{select_sql} - ON CONFLICT (uncle_hash, index) - DO UPDATE SET hash = EXCLUDED.hash - WHERE EXCLUDED.hash <> transaction_forks.hash - RETURNING uncle_hash, hash - """ - - with {:ok, %Postgrex.Result{columns: ["uncle_hash", "hash"], command: :insert, rows: rows}} <- - SQL.query( - repo, - insert_sql, - parameters, - timeout: timeout - ) do - derived_transaction_forks = Enum.map(rows, fn [uncle_hash, hash] -> %{uncle_hash: uncle_hash, hash: hash} end) + with {:ok, %{insert_transaction_forks: rows}} <- repo.transaction(multi, timeout: timeout) do + derived_transaction_forks = Enum.map(rows, &Map.take(&1, [:uncle_hash, :hash])) {:ok, derived_transaction_forks} end @@ -214,23 +197,32 @@ defmodule Explorer.Chain.Import.Runner.Blocks do }) do query = where_forked - |> update( - set: [ - block_hash: nil, - block_number: nil, - gas_used: nil, - cumulative_gas_used: nil, - index: nil, - internal_transactions_indexed_at: nil, - status: nil, - error: nil, - updated_at: ^updated_at - ] + # Enforce Transaction ShareLocks order (see docs: sharelocks.md) + |> order_by(asc: :hash) + |> lock("FOR UPDATE") + + update_query = + from(t in Transaction, + join: s in subquery(query), + on: t.hash == s.hash, + update: [ + set: [ + block_hash: nil, + block_number: nil, + gas_used: nil, + cumulative_gas_used: nil, + index: nil, + internal_transactions_indexed_at: nil, + status: nil, + error: nil, + updated_at: ^updated_at + ] + ], + select: t.hash ) - |> select([:hash]) try do - {_, result} = repo.update_all(query, [], timeout: timeout) + {_, result} = repo.update_all(update_query, [], timeout: timeout) {:ok, result} rescue @@ -247,8 +239,8 @@ defmodule Explorer.Chain.Import.Runner.Blocks do defp insert(repo, changes_list, %{timeout: timeout, timestamps: timestamps} = options) when is_list(changes_list) do on_conflict = Map.get_lazy(options, :on_conflict, &default_on_conflict/0) - # order so that row ShareLocks are grabbed in a consistent order - ordered_changes_list = Enum.sort_by(changes_list, &{&1.number, &1.hash}) + # Enforce Block ShareLocks order (see docs: sharelocks.md) + ordered_changes_list = Enum.sort_by(changes_list, & &1.hash) Import.insert_changes_list( repo, @@ -316,17 +308,18 @@ defmodule Explorer.Chain.Import.Runner.Blocks do from( block in Block, where: block.number in ^ordered_consensus_block_number, - update: [ - set: [ - consensus: false, - updated_at: ^updated_at - ] - ], - select: [:hash, :number] + # Enforce Block ShareLocks order (see docs: sharelocks.md) + order_by: [asc: block.hash], + lock: "FOR UPDATE" ) try do - {_, result} = repo.update_all(query, [], timeout: timeout) + {_, result} = + repo.update_all( + from(b in Block, join: s in subquery(query), on: b.hash == s.hash, select: [:hash, :number]), + [set: [consensus: false, updated_at: updated_at]], + timeout: timeout + ) {:ok, result} rescue @@ -342,17 +335,18 @@ defmodule Explorer.Chain.Import.Runner.Blocks do query = from( block in where_invalid_neighbour, - update: [ - set: [ - consensus: false, - updated_at: ^updated_at - ] - ], - select: [:hash, :number] + # Enforce Block ShareLocks order (see docs: sharelocks.md) + order_by: [asc: block.hash], + lock: "FOR UPDATE" ) try do - {_, result} = repo.update_all(query, [], timeout: timeout) + {_, result} = + repo.update_all( + from(b in Block, join: s in subquery(query), on: b.hash == s.hash, select: [:hash, :number]), + [set: [consensus: false, updated_at: updated_at]], + timeout: timeout + ) {:ok, result} rescue @@ -498,13 +492,12 @@ defmodule Explorer.Chain.Import.Runner.Blocks do from(address_token_balance in Address.TokenBalance, where: address_token_balance.block_number in ^ordered_consensus_block_numbers, select: map(address_token_balance, [:address_hash, :token_contract_address_hash, :block_number]), - # MUST match order in `Explorer.Chain.Import.Runner.Address.TokenBalances.insert` to prevent ShareLock ordering deadlocks. + # Enforce TokenBalance ShareLocks order (see docs: sharelocks.md) order_by: [ address_token_balance.address_hash, address_token_balance.token_contract_address_hash, address_token_balance.block_number ], - # ensures rows remains locked while outer query is joining to it lock: "FOR UPDATE" ) @@ -536,12 +529,11 @@ defmodule Explorer.Chain.Import.Runner.Blocks do from(address_current_token_balance in Address.CurrentTokenBalance, where: address_current_token_balance.block_number in ^ordered_consensus_block_numbers, select: map(address_current_token_balance, [:address_hash, :token_contract_address_hash]), - # MUST match order in `Explorer.Chain.Import.Runner.Address.CurrentTokenBalances.insert` to prevent ShareLock ordering deadlocks. + # Enforce CurrentTokenBalance ShareLocks order (see docs: sharelocks.md) order_by: [ address_current_token_balance.address_hash, address_current_token_balance.token_contract_address_hash ], - # ensures row remains locked while outer query is joining to it lock: "FOR UPDATE" ) @@ -575,7 +567,6 @@ defmodule Explorer.Chain.Import.Runner.Blocks do defp derive_address_current_token_balances(_, [], _), do: {:ok, []} - # sobelow_skip ["SQL.Query"] defp derive_address_current_token_balances(repo, deleted_address_current_token_balances, %{timeout: timeout}) when is_list(deleted_address_current_token_balances) do initial_query = @@ -609,54 +600,51 @@ defmodule Explorer.Chain.Import.Runner.Blocks do address_token_balance.address_hash == new_current_token_balance.address_hash and address_token_balance.token_contract_address_hash == new_current_token_balance.token_contract_address_hash and address_token_balance.block_number == new_current_token_balance.block_number, - select: { - new_current_token_balance.address_hash, - new_current_token_balance.token_contract_address_hash, - new_current_token_balance.block_number, - address_token_balance.value, - over(min(address_token_balance.inserted_at), :w), - over(max(address_token_balance.updated_at), :w) + select: %{ + address_hash: new_current_token_balance.address_hash, + token_contract_address_hash: new_current_token_balance.token_contract_address_hash, + block_number: new_current_token_balance.block_number, + value: address_token_balance.value, + inserted_at: over(min(address_token_balance.inserted_at), :w), + updated_at: over(max(address_token_balance.updated_at), :w) }, - # Prevent ShareLock deadlock by matching order of `Explorer.Chain.Import.Runner.Address.CurrentTokenBalances.insert` - order_by: [new_current_token_balance.address_hash, new_current_token_balance.token_contract_address_hash], windows: [ w: [partition_by: [address_token_balance.address_hash, address_token_balance.token_contract_address_hash]] ] ) - {select_sql, parameters} = SQL.to_sql(:all, repo, new_current_token_balance_query) - - # No `ON CONFLICT` because `delete_address_current_token_balances` should have removed any conflicts. - insert_sql = """ - INSERT INTO address_current_token_balances (address_hash, token_contract_address_hash, block_number, value, inserted_at, updated_at) - #{select_sql} - RETURNING address_hash, token_contract_address_hash, block_number, value - """ - - with {:ok, - %Postgrex.Result{ - columns: [ - "address_hash", - "token_contract_address_hash", - "block_number", - # needed for `update_tokens_holder_count` - "value" - ], - command: :insert, - rows: rows - }} <- SQL.query(repo, insert_sql, parameters, timeout: timeout) do + multi = + Multi.new() + |> Multi.run(:new_current_token_balance, fn repo, _ -> + new_current_token_balances = repo.all(new_current_token_balance_query) + {:ok, new_current_token_balances} + end) + |> Multi.run( + :insert_new_current_token_balance, + fn repo, %{new_current_token_balance: new_current_token_balance} -> + # Enforce CurrentTokenBalance ShareLocks order (see docs: sharelocks.md) + ordered_current_token_balance = + Enum.sort_by( + new_current_token_balance, + &{&1.address_hash, &1.token_contract_address_hash} + ) + + {_total, result} = + repo.insert_all( + Address.CurrentTokenBalance, + ordered_current_token_balance, + # No `ON CONFLICT` because `delete_address_current_token_balances` + # should have removed any conflicts. + returning: [:address_hash, :token_contract_address_hash, :block_number, :value] + ) + + {:ok, result} + end + ) + + with {:ok, %{insert_new_current_token_balance: rows}} <- repo.transaction(multi, timeout: timeout) do derived_address_current_token_balances = - Enum.map(rows, fn [address_hash_bytes, token_contract_address_hash_bytes, block_number, value] -> - {:ok, address_hash} = Hash.Address.load(address_hash_bytes) - {:ok, token_contract_address_hash} = Hash.Address.load(token_contract_address_hash_bytes) - - %{ - address_hash: address_hash, - token_contract_address_hash: token_contract_address_hash, - block_number: block_number, - value: value - } - end) + Enum.map(rows, &Map.take(&1, [:address_hash, :token_contract_address_hash, :block_number, :value])) {:ok, derived_address_current_token_balances} end @@ -677,11 +665,24 @@ defmodule Explorer.Chain.Import.Runner.Blocks do query = from(reward in Reward, inner_join: block in assoc(reward, :block), - where: block.hash in ^hashes or block.number in ^numbers + where: block.hash in ^hashes or block.number in ^numbers, + # Enforce Reward ShareLocks order (see docs: sharelocks.md) + order_by: [asc: :address_hash, asc: :address_type, asc: :block_hash], + # NOTE: find a better way to know the alias that ecto gives to token + lock: "FOR UPDATE OF b0" + ) + + delete_query = + from(r in Reward, + join: s in subquery(query), + on: + r.address_hash == s.address_hash and + r.address_type == s.address_type and + r.block_hash == s.block_hash ) try do - {count, nil} = repo.delete_all(query, timeout: timeout) + {count, nil} = repo.delete_all(delete_query, timeout: timeout) {:ok, count} rescue @@ -692,32 +693,73 @@ defmodule Explorer.Chain.Import.Runner.Blocks do defp update_block_second_degree_relations(repo, blocks, %{timeout: timeout, timestamps: %{updated_at: updated_at}}) when is_list(blocks) do - ordered_uncle_hashes = + uncle_hashes = blocks |> MapSet.new(& &1.hash) - |> Enum.sort() + |> MapSet.to_list() query = from( bsdr in Block.SecondDegreeRelation, - where: bsdr.uncle_hash in ^ordered_uncle_hashes, - update: [ - set: [ - uncle_fetched_at: ^updated_at - ] - ] + where: bsdr.uncle_hash in ^uncle_hashes, + # Enforce SeconDegreeRelation ShareLocks order (see docs: sharelocks.md) + order_by: [asc: :nephew_hash, asc: :uncle_hash], + lock: "FOR UPDATE" + ) + + update_query = + from( + b in Block.SecondDegreeRelation, + join: s in subquery(query), + on: b.nephew_hash == s.nephew_hash and b.uncle_hash == s.uncle_hash, + update: [set: [uncle_fetched_at: ^updated_at]] ) try do - {_, result} = repo.update_all(query, [], timeout: timeout) + {_, result} = repo.update_all(update_query, [], timeout: timeout) {:ok, result} rescue postgrex_error in Postgrex.Error -> - {:error, %{exception: postgrex_error, uncle_hashes: ordered_uncle_hashes}} + {:error, %{exception: postgrex_error, uncle_hashes: uncle_hashes}} end end + defp update_internal_transaction_block_number(repo, blocks) when is_list(blocks) do + blocks_hashes = Enum.map(blocks, & &1.hash) + + query = + from( + internal_transaction in InternalTransaction, + join: transaction in Transaction, + on: internal_transaction.transaction_hash == transaction.hash, + join: block in Block, + on: block.hash == transaction.block_hash, + where: block.hash in ^blocks_hashes, + select: %{ + transaction_hash: internal_transaction.transaction_hash, + index: internal_transaction.index, + block_number: block.number + }, + # Enforce InternalTransaction ShareLocks order (see docs: sharelocks.md) + order_by: [asc: :transaction_hash, asc: :index], + # NOTE: find a better way to know the alias that ecto gives to internal_transaction + lock: "FOR UPDATE OF i0" + ) + + update_query = + from( + i in InternalTransaction, + join: s in subquery(query), + on: i.transaction_hash == s.transaction_hash and i.index == s.index, + update: [set: [block_number: s.block_number]] + ) + + {total, _} = repo.update_all(update_query, []) + + {:ok, total} + end + defp where_forked(blocks_changes) when is_list(blocks_changes) do initial = from(t in Transaction, where: false) diff --git a/apps/explorer/lib/explorer/chain/import/runner/internal_transactions.ex b/apps/explorer/lib/explorer/chain/import/runner/internal_transactions.ex index 50529e8ce8..c7422f074b 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/internal_transactions.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/internal_transactions.ex @@ -70,7 +70,7 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactions do when is_list(changes_list) do on_conflict = Map.get_lazy(options, :on_conflict, &default_on_conflict/0) - # order so that row ShareLocks are grabbed in a consistent order + # Enforce InternalTransaction ShareLocks order (see docs: sharelocks.md) ordered_changes_list = Enum.sort_by(changes_list, &{&1.transaction_hash, &1.index}) final_changes_list = reject_pending_transactions(ordered_changes_list, repo) @@ -149,16 +149,26 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactions do timestamps: timestamps }) when is_list(internal_transactions) do - ordered_transaction_hashes = + transaction_hashes = internal_transactions |> MapSet.new(& &1.transaction_hash) - |> Enum.sort() + |> MapSet.to_list() query = from( t in Transaction, - where: t.hash in ^ordered_transaction_hashes, + where: t.hash in ^transaction_hashes, where: not is_nil(t.block_hash), + # Enforce Transaction ShareLocks order (see docs: sharelocks.md) + order_by: t.hash, + lock: "FOR UPDATE" + ) + + update_query = + from( + t in Transaction, + join: s in subquery(query), + on: t.hash == s.hash, update: [ set: [ internal_transactions_indexed_at: ^timestamps.updated_at, @@ -184,12 +194,12 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactions do ) try do - {_transaction_count, result} = repo.update_all(query, [], timeout: timeout) + {_transaction_count, result} = repo.update_all(update_query, [], timeout: timeout) {:ok, result} rescue postgrex_error in Postgrex.Error -> - {:error, %{exception: postgrex_error, transaction_hashes: ordered_transaction_hashes}} + {:error, %{exception: postgrex_error, transaction_hashes: transaction_hashes}} end end diff --git a/apps/explorer/lib/explorer/chain/import/runner/internal_transactions_indexed_at_blocks.ex b/apps/explorer/lib/explorer/chain/import/runner/internal_transactions_indexed_at_blocks.ex index 002a607c0c..2726670a30 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/internal_transactions_indexed_at_blocks.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/internal_transactions_indexed_at_blocks.ex @@ -49,36 +49,36 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactionsIndexedAtBlocks do defp update_blocks(_repo, [], %{}), do: {:ok, []} - defp update_blocks(repo, block_numbers, %{ + defp update_blocks(repo, changes_list, %{ timeout: timeout, timestamps: timestamps }) - when is_list(block_numbers) do - ordered_block_numbers = - block_numbers - |> Enum.map(fn %{number: number} -> number end) - |> Enum.sort() + when is_list(changes_list) do + block_numbers = Enum.map(changes_list, fn %{number: number} -> number end) query = from( b in Block, - where: b.number in ^ordered_block_numbers and b.consensus, - update: [ - set: [ - internal_transactions_indexed_at: ^timestamps.updated_at - ] - ] + where: b.number in ^block_numbers and b.consensus, + # Enforce Block ShareLocks order (see docs: sharelocks.md) + order_by: [asc: b.hash], + lock: "FOR UPDATE" ) - block_count = Enum.count(ordered_block_numbers) + block_count = Enum.count(block_numbers) try do - {^block_count, result} = repo.update_all(query, [], timeout: timeout) + {^block_count, result} = + repo.update_all( + from(b in Block, join: s in subquery(query), on: b.hash == s.hash), + [set: [internal_transactions_indexed_at: timestamps.updated_at]], + timeout: timeout + ) {:ok, result} rescue postgrex_error in Postgrex.Error -> - {:error, %{exception: postgrex_error, block_numbers: ordered_block_numbers}} + {:error, %{exception: postgrex_error, block_numbers: block_numbers}} end end end diff --git a/apps/explorer/lib/explorer/chain/import/runner/logs.ex b/apps/explorer/lib/explorer/chain/import/runner/logs.ex index 086a1cc6b2..ddac92a647 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/logs.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/logs.ex @@ -58,7 +58,7 @@ defmodule Explorer.Chain.Import.Runner.Logs do defp insert(repo, changes_list, %{timeout: timeout, timestamps: timestamps} = options) when is_list(changes_list) do on_conflict = Map.get_lazy(options, :on_conflict, &default_on_conflict/0) - # order so that row ShareLocks are grabbed in a consistent order + # Enforce Log ShareLocks order (see docs: sharelocks.md) ordered_changes_list = Enum.sort_by(changes_list, &{&1.transaction_hash, &1.index}) {:ok, _} = diff --git a/apps/explorer/lib/explorer/chain/import/runner/staking_pools.ex b/apps/explorer/lib/explorer/chain/import/runner/staking_pools.ex index ca83fa62ee..13ba8cb5f9 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/staking_pools.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/staking_pools.ex @@ -62,16 +62,18 @@ defmodule Explorer.Chain.Import.Runner.StakingPools do from( pool in StakingPool, where: pool.staking_address_hash not in ^addresses, - update: [ - set: [ - is_deleted: true, - is_active: false - ] - ] + # Enforce StackingPool ShareLocks order (see docs: sharelocks.md) + order_by: pool.staking_address_hash, + lock: "FOR UPDATE" ) try do - {_, result} = repo.update_all(query, [], timeout: timeout) + {_, result} = + repo.update_all( + from(p in StakingPool, join: s in subquery(query), on: p.staking_address_hash == s.staking_address_hash), + [set: [is_deleted: true, is_active: false]], + timeout: timeout + ) {:ok, result} rescue @@ -90,10 +92,13 @@ defmodule Explorer.Chain.Import.Runner.StakingPools do defp insert(repo, changes_list, %{timeout: timeout, timestamps: timestamps} = options) when is_list(changes_list) do on_conflict = Map.get_lazy(options, :on_conflict, &default_on_conflict/0) + # Enforce StackingPool ShareLocks order (see docs: sharelocks.md) + ordered_changes_list = Enum.sort_by(changes_list, & &1.staking_address_hash) + {:ok, _} = Import.insert_changes_list( repo, - changes_list, + ordered_changes_list, conflict_target: :staking_address_hash, on_conflict: on_conflict, for: StakingPool, @@ -142,6 +147,15 @@ defmodule Explorer.Chain.Import.Runner.StakingPools do from( p in StakingPool, where: p.is_active == true, + # Enforce StackingPool ShareLocks order (see docs: sharelocks.md) + order_by: p.staking_address_hash, + lock: "FOR UPDATE" + ) + + update_query = + from(p in StakingPool, + join: s in subquery(query), + on: p.staking_address_hash == s.staking_address_hash, update: [ set: [ staked_ratio: p.staked_amount / ^total * 100, @@ -150,7 +164,8 @@ defmodule Explorer.Chain.Import.Runner.StakingPools do ] ) - {count, _} = repo.update_all(query, [], timeout: timeout) + {count, _} = repo.update_all(update_query, [], timeout: timeout) + {:ok, count} else {:ok, 1} diff --git a/apps/explorer/lib/explorer/chain/import/runner/staking_pools_delegators.ex b/apps/explorer/lib/explorer/chain/import/runner/staking_pools_delegators.ex index 420882a833..4677aeee04 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/staking_pools_delegators.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/staking_pools_delegators.ex @@ -59,10 +59,13 @@ defmodule Explorer.Chain.Import.Runner.StakingPoolsDelegators do defp insert(repo, changes_list, %{timeout: timeout, timestamps: timestamps} = options) when is_list(changes_list) do on_conflict = Map.get_lazy(options, :on_conflict, &default_on_conflict/0) + # Enforce StackingPoolDelegator ShareLocks order (see docs: sharelocks.md) + ordered_changes_list = Enum.sort_by(changes_list, &{&1.delegator_address_hash, &1.pool_address_hash}) + {:ok, _} = Import.insert_changes_list( repo, - changes_list, + ordered_changes_list, conflict_target: [:pool_address_hash, :delegator_address_hash], on_conflict: on_conflict, for: StakingPoolsDelegator, diff --git a/apps/explorer/lib/explorer/chain/import/runner/token_transfers.ex b/apps/explorer/lib/explorer/chain/import/runner/token_transfers.ex index fb98b76dee..cc88389f4c 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/token_transfers.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/token_transfers.ex @@ -54,7 +54,7 @@ defmodule Explorer.Chain.Import.Runner.TokenTransfers do def insert(repo, changes_list, %{timeout: timeout, timestamps: timestamps} = options) when is_list(changes_list) do on_conflict = Map.get_lazy(options, :on_conflict, &default_on_conflict/0) - # order so that row ShareLocks are grabbed in a consistent order + # Enforce TokenTransfer ShareLocks order (see docs: sharelocks.md) ordered_changes_list = Enum.sort_by(changes_list, &{&1.transaction_hash, &1.log_index}) {:ok, _} = diff --git a/apps/explorer/lib/explorer/chain/import/runner/tokens.ex b/apps/explorer/lib/explorer/chain/import/runner/tokens.ex index dcc8d0cb89..22621b022c 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/tokens.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/tokens.ex @@ -7,7 +7,6 @@ defmodule Explorer.Chain.Import.Runner.Tokens do import Ecto.Query, only: [from: 2] - alias Ecto.Adapters.SQL alias Ecto.{Multi, Repo} alias Explorer.Chain.{Hash, Import, Token} @@ -22,10 +21,59 @@ defmodule Explorer.Chain.Import.Runner.Tokens do @type holder_count :: non_neg_integer() @type token_holder_count :: %{contract_address_hash: Hash.Address.t(), count: holder_count()} - def update_holder_counts_with_deltas(repo, token_holder_count_deltas, options) do - parameters = token_holder_count_deltas_to_parameters(token_holder_count_deltas) + def update_holder_counts_with_deltas(repo, token_holder_count_deltas, %{ + timeout: timeout, + timestamps: %{updated_at: updated_at} + }) do + {hashes, deltas} = + token_holder_count_deltas + |> Enum.map(fn %{contract_address_hash: contract_address_hash, delta: delta} -> + {:ok, contract_address_hash_bytes} = Hash.Address.dump(contract_address_hash) + {contract_address_hash_bytes, delta} + end) + |> Enum.unzip() + + query = + from( + token in Token, + join: + deltas in fragment( + "(SELECT unnest(?::bytea[]) as contract_address_hash, unnest(?::bigint[]) as delta)", + ^hashes, + ^deltas + ), + on: token.contract_address_hash == deltas.contract_address_hash, + select: %{ + contract_address_hash: token.contract_address_hash, + delta: deltas.delta + }, + where: not is_nil(token.holder_count), + # Enforce Token ShareLocks order (see docs: sharelocks.md) + order_by: token.contract_address_hash, + # NOTE: find a better way to know the alias that ecto gives to token + lock: "FOR UPDATE OF t0" + ) - update_holder_counts_with_parameters(repo, parameters, options) + update_query = + from( + t in Token, + join: s in subquery(query), + on: t.contract_address_hash == s.contract_address_hash, + update: [ + set: [ + holder_count: t.holder_count + s.delta, + updated_at: ^updated_at + ] + ], + select: %{ + contract_address_hash: t.contract_address_hash, + holder_count: t.holder_count + } + ) + + {_total, result} = repo.update_all(update_query, [], timeout: timeout) + + {:ok, result} end @impl Import.Runner @@ -71,7 +119,7 @@ defmodule Explorer.Chain.Import.Runner.Tokens do changes_list # brand new tokens start with no holders |> Stream.map(&Map.put_new(&1, :holder_count, 0)) - # order so that row ShareLocks are grabbed in a consistent order + # Enforce Token ShareLocks order (see docs: sharelocks.md) |> Enum.sort_by(& &1.contract_address_hash) {:ok, _} = @@ -117,69 +165,4 @@ defmodule Explorer.Chain.Import.Runner.Tokens do ) ) end - - defp token_holder_count_deltas_to_parameters(token_holder_count_deltas) when is_list(token_holder_count_deltas) do - Enum.flat_map(token_holder_count_deltas, fn - %{contract_address_hash: contract_address_hash, delta: delta} -> - {:ok, contract_address_hash_bytes} = Hash.Address.dump(contract_address_hash) - [contract_address_hash_bytes, delta] - end) - end - - defp update_holder_counts_with_parameters(_, [], _), do: {:ok, []} - - # sobelow_skip ["SQL.Query"] - defp update_holder_counts_with_parameters(repo, parameters, %{timeout: timeout, timestamps: %{updated_at: updated_at}}) - when is_list(parameters) do - update_sql = update_holder_counts_sql(parameters) - - with {:ok, %Postgrex.Result{columns: ["contract_address_hash", "holder_count"], command: :update, rows: rows}} <- - SQL.query(repo, update_sql, [updated_at | parameters], timeout: timeout) do - update_token_holder_counts = - Enum.map(rows, fn [contract_address_hash_bytes, holder_count] -> - {:ok, contract_address_hash} = Hash.Address.cast(contract_address_hash_bytes) - %{contract_address_hash: contract_address_hash, holder_count: holder_count} - end) - - {:ok, update_token_holder_counts} - end - end - - defp update_holder_counts_sql(parameters) when is_list(parameters) do - parameters - |> Enum.count() - |> div(2) - |> update_holder_counts_sql() - end - - defp update_holder_counts_sql(row_count) when is_integer(row_count) do - parameters_sql = - update_holder_counts_parameters_sql( - row_count, - # skip $1 as it is used for the common `updated_at` timestamp - 2 - ) - - """ - UPDATE tokens - SET holder_count = holder_count + holder_counts.delta, - updated_at = $1 - FROM ( - VALUES - #{parameters_sql} - ) AS holder_counts(contract_address_hash, delta) - WHERE tokens.contract_address_hash = holder_counts.contract_address_hash AND - holder_count IS NOT NULL - RETURNING tokens.contract_address_hash, tokens.holder_count - """ - end - - defp update_holder_counts_parameters_sql(row_count, start) when is_integer(row_count) do - Enum.map_join(0..(row_count - 1), ",\n ", fn i -> - contract_address_hash_parameter_number = 2 * i + start - holder_count_number = contract_address_hash_parameter_number + 1 - - "($#{contract_address_hash_parameter_number}::bytea, $#{holder_count_number}::bigint)" - end) - end end diff --git a/apps/explorer/lib/explorer/chain/import/runner/transaction/forks.ex b/apps/explorer/lib/explorer/chain/import/runner/transaction/forks.ex index d005b5115b..99e9c275dd 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/transaction/forks.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/transaction/forks.ex @@ -58,8 +58,8 @@ defmodule Explorer.Chain.Import.Runner.Transaction.Forks do defp insert(repo, changes_list, %{timeout: timeout, timestamps: timestamps} = options) when is_list(changes_list) do on_conflict = Map.get_lazy(options, :on_conflict, &default_on_conflict/0) - # order so that row ShareLocks are grabbed in a consistent order - ordered_changes_list = Enum.sort_by(changes_list, &{&1.uncle_hash, &1.hash}) + # Enforce Fork ShareLocks order (see docs: sharelocks.md) + ordered_changes_list = Enum.sort_by(changes_list, &{&1.uncle_hash, &1.index}) Import.insert_changes_list( repo, diff --git a/apps/explorer/lib/explorer/chain/import/runner/transactions.ex b/apps/explorer/lib/explorer/chain/import/runner/transactions.ex index 570d934218..89c89aae4d 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/transactions.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/transactions.ex @@ -81,7 +81,7 @@ defmodule Explorer.Chain.Import.Runner.Transactions do ordered_changes_list = changes_list |> put_internal_transactions_indexed_at(inserted_at, token_transfer_transaction_hash_set) - # order so that row ShareLocks are grabbed in a consistent order + # Enforce Transaction ShareLocks order (see docs: sharelocks.md) |> Enum.sort_by(& &1.hash) Import.insert_changes_list( @@ -191,36 +191,38 @@ defmodule Explorer.Chain.Import.Runner.Transactions do timestamps: %{updated_at: updated_at} }) when is_list(transactions) do - ordered_block_hashes = + block_hashes = transactions |> Enum.filter(fn %{block_hash: block_hash, old_block_hash: old_block_hash} -> not is_nil(old_block_hash) and block_hash != old_block_hash end) |> MapSet.new(& &1.old_block_hash) - |> Enum.sort() + |> MapSet.to_list() - if Enum.empty?(ordered_block_hashes) do + if Enum.empty?(block_hashes) do {:ok, []} else query = from( block in Block, - where: block.hash in ^ordered_block_hashes, - update: [ - set: [ - consensus: false, - updated_at: ^updated_at - ] - ] + where: block.hash in ^block_hashes, + # Enforce Block ShareLocks order (see docs: sharelocks.md) + order_by: [asc: block.hash], + lock: "FOR UPDATE" ) try do - {_, result} = repo.update_all(query, [], timeout: timeout) + {_, result} = + repo.update_all( + from(b in Block, join: s in subquery(query), on: b.hash == s.hash), + [set: [consensus: false, updated_at: updated_at]], + timeout: timeout + ) {:ok, result} rescue postgrex_error in Postgrex.Error -> - {:error, %{exception: postgrex_error, block_hashes: ordered_block_hashes}} + {:error, %{exception: postgrex_error, block_hashes: block_hashes}} end end end diff --git a/apps/explorer/lib/explorer/chain_spec/parity/importer.ex b/apps/explorer/lib/explorer/chain_spec/parity/importer.ex index fa16ee0739..9f3585f822 100644 --- a/apps/explorer/lib/explorer/chain_spec/parity/importer.ex +++ b/apps/explorer/lib/explorer/chain_spec/parity/importer.ex @@ -12,6 +12,8 @@ defmodule Explorer.ChainSpec.Parity.Importer do alias Explorer.ChainSpec.GenesisData alias Explorer.ChainSpec.POA.Importer, as: PoaEmissionImporter + import Ecto.Query + @max_block_number :infinity def import_emission_rewards(chain_spec) do @@ -46,8 +48,27 @@ defmodule Explorer.ChainSpec.Parity.Importer do defp import_rewards_from_chain_spec(chain_spec) do rewards = emission_rewards(chain_spec) - {_, nil} = Repo.delete_all(EmissionReward) - {_, nil} = Repo.insert_all(EmissionReward, rewards) + inner_delete_query = + from( + emission_reward in EmissionReward, + # Enforce EmissionReward ShareLocks order (see docs: sharelocks.md) + order_by: emission_reward.block_range, + lock: "FOR UPDATE" + ) + + delete_query = + from( + e in EmissionReward, + join: s in subquery(inner_delete_query), + # we join on reward because it's faster and we have to delete them all anyway + on: e.reward == s.reward + ) + + # Enforce EmissionReward ShareLocks order (see docs: sharelocks.md) + ordered_rewards = Enum.sort_by(rewards, & &1.block_range) + + {_, nil} = Repo.delete_all(delete_query) + {_, nil} = Repo.insert_all(EmissionReward, ordered_rewards) end def genesis_coin_balances(chain_spec) do diff --git a/apps/explorer/lib/explorer/chain_spec/poa/importer.ex b/apps/explorer/lib/explorer/chain_spec/poa/importer.ex index 03935d85b2..33d2e9199a 100644 --- a/apps/explorer/lib/explorer/chain_spec/poa/importer.ex +++ b/apps/explorer/lib/explorer/chain_spec/poa/importer.ex @@ -11,6 +11,8 @@ defmodule Explorer.ChainSpec.POA.Importer do alias Explorer.Chain.Block.{EmissionReward, Range} alias Explorer.ChainSpec.GenesisData + import Ecto.Query + @block_reward_amount_abi %{ "type" => "function", "stateMutability" => "view", @@ -51,8 +53,27 @@ defmodule Explorer.ChainSpec.POA.Importer do } ] - {_, nil} = Repo.delete_all(EmissionReward) - {_, nil} = Repo.insert_all(EmissionReward, rewards) + inner_delete_query = + from( + emission_reward in EmissionReward, + # Enforce EmissionReward ShareLocks order (see docs: sharelocks.md) + order_by: emission_reward.block_range, + lock: "FOR UPDATE" + ) + + delete_query = + from( + e in EmissionReward, + join: s in subquery(inner_delete_query), + # we join on reward because it's faster and we have to delete them all anyway + on: e.reward == s.reward + ) + + # Enforce EmissionReward ShareLocks order (see docs: sharelocks.md) + ordered_rewards = Enum.sort_by(rewards, & &1.block_range) + + {_, nil} = Repo.delete_all(delete_query) + {_, nil} = Repo.insert_all(EmissionReward, ordered_rewards) end end diff --git a/apps/explorer/lib/explorer/market/market.ex b/apps/explorer/lib/explorer/market/market.ex index 3adb7316fd..24b15f5056 100644 --- a/apps/explorer/lib/explorer/market/market.ex +++ b/apps/explorer/lib/explorer/market/market.ex @@ -41,9 +41,12 @@ defmodule Explorer.Market do @doc false def bulk_insert_history(records) do records_without_zeroes = - Enum.reject(records, fn item -> + records + |> Enum.reject(fn item -> Decimal.equal?(item.closing_price, 0) && Decimal.equal?(item.opening_price, 0) end) + # Enforce MarketHistory ShareLocks order (see docs: sharelocks.md) + |> Enum.sort_by(& &1.date) Repo.insert_all(MarketHistory, records_without_zeroes, on_conflict: :nothing, conflict_target: [:date]) end diff --git a/apps/explorer/lib/explorer/validator/metadata_importer.ex b/apps/explorer/lib/explorer/validator/metadata_importer.ex index 3bf799d51c..5c6757ffbd 100644 --- a/apps/explorer/lib/explorer/validator/metadata_importer.ex +++ b/apps/explorer/lib/explorer/validator/metadata_importer.ex @@ -8,7 +8,10 @@ defmodule Explorer.Validator.MetadataImporter do import Ecto.Query, only: [from: 2] def import_metadata(metadata_maps) do - Repo.transaction(fn -> Enum.each(metadata_maps, &upsert_validator_metadata(&1)) end) + # Enforce Name ShareLocks order (see docs: sharelocks.md) + ordered_metadata_maps = Enum.sort_by(metadata_maps, &{&1.address_hash, &1.name}) + + Repo.transaction(fn -> Enum.each(ordered_metadata_maps, &upsert_validator_metadata(&1)) end) end defp upsert_validator_metadata(validator_changeset) do diff --git a/apps/indexer/lib/indexer/temporary/blocks_transactions_mismatch.ex b/apps/indexer/lib/indexer/temporary/blocks_transactions_mismatch.ex index ec1bb7f7b9..a1c9f5befc 100644 --- a/apps/indexer/lib/indexer/temporary/blocks_transactions_mismatch.ex +++ b/apps/indexer/lib/indexer/temporary/blocks_transactions_mismatch.ex @@ -105,19 +105,15 @@ defmodule Indexer.Temporary.BlocksTransactionsMismatch do end) unless Enum.empty?(matching_blocks_data) do - hashes = Enum.map(matching_blocks_data, fn {hash, _trans_num} -> hash end) - - Block - |> where([block], block.hash in ^hashes) - |> Repo.update_all(set: [refetch_needed: false]) + matching_blocks_data + |> Enum.map(fn {hash, _trans_num} -> hash end) + |> update_in_order(refetch_needed: false) end unless Enum.empty?(unmatching_blocks_data) do - hashes = Enum.map(unmatching_blocks_data, fn {hash, _trans_num} -> hash end) - - Block - |> where([block], block.hash in ^hashes) - |> Repo.update_all(set: [refetch_needed: false, consensus: false]) + unmatching_blocks_data + |> Enum.map(fn {hash, _trans_num} -> hash end) + |> update_in_order(refetch_needed: false, consensus: false) end if Enum.empty?(missing_blocks_data) do @@ -126,4 +122,19 @@ defmodule Indexer.Temporary.BlocksTransactionsMismatch do {:retry, missing_blocks_data} end end + + defp update_in_order(hashes, fields_to_set) do + query = + from(block in Block, + where: block.hash in ^hashes, + # Enforce Block ShareLocks order (see docs: sharelocks.md) + order_by: [asc: block.hash], + lock: "FOR UPDATE" + ) + + Repo.update_all( + from(b in Block, join: s in subquery(query), on: b.hash == s.hash), + set: fields_to_set + ) + end end diff --git a/docs/sharelocks.md b/docs/sharelocks.md new file mode 100644 index 0000000000..ebcc3570a6 --- /dev/null +++ b/docs/sharelocks.md @@ -0,0 +1,88 @@ + + +## ShareLocks + +ShareLock is the row-level locking mechanism used internally by PostgreSQL. + +### Deadlocks and prevention + +When several DB transactions are acting on multiple rows of the same table, it's +possible to incur in a deadlock and so into an error. +This can be prevented by enforcing the same consistent order of lock aquisition +on *all* the transactions performing `INSERT`, `UPDATE` or `DELETE` on a given table. + +### Imposing the lock acquisition order with Ecto + +When `INSERT`ing a list of rows Postgres will respect the order in which they +appear in the query, so the reordering can happen beforehand. + +For example, this will work: +```elixir +entries = [...] + +ordered_entries = Enum.sort_by(entries, & &1.id) + +Repo.insert_all(__MODULE__, ordered_entries) +``` + +Performing `UPDATE`s is trickier because there is no `ORDER BY` clause. +The solution to this is to `JOIN` on a subquery that `SELECT`s with the option `FOR UPDATE`. + +Using Ecto this can be done, for example, like this: +```elixir +query = + from( + entry in Entry, + where: not is_nil(entry.value), + order_by: entry.id, + lock: "FOR UPDATE" + ) + +Repo.update_all( + from(e in Entry, join: s in subquery(query), on: e.id == s.id), + [set: [value: nil]], + timeout: timeout) +``` + +`DELETE` has the same quircks as `UPDATE` and it is too solved in the same way. + +For example: +```elixir +query = + from( + entry in Entry, + where: is_nil(entry.value), + order_by: entry.id, + lock: "FOR UPDATE" + ) + +Repo.delete_all(from(e in Entry, join: s in subquery(query), on: e.id == s.id)) +``` + +### Order used in Explorer's tables + +This is a complete list of the ordering currently in use on each table. +Note that this should always be enforced because as long as there is one DB +transaction performing in a different order there is the possibility of a deadlock. + +| schema module | table name | ordered by | +|---------------|------------|------------| +| Explorer.Chain.Address | addresses | asc: :hash | +| Explorer.Chain.Address.CoinBalance | address_coin_balances | [asc: :address_hash, asc: :block_number] | +| Explorer.Chain.Address.CurrentTokenBalance | address_current_token_balances | [asc: :address_hash, asc: :token_contract_address_hash] | +| Explorer.Chain.Address.Name | address_names | [asc: :address_hash, asc: :name] | +| Explorer.Chain.Address.TokenBalance | address_token_balances | [asc: :address_hash, asc: :token_contract_address_hash, asc: :block_number] | +| Explorer.Chain.Block | blocks | asc: :hash | +| Explorer.Chain.Block.EmissionReward | emission_rewards | asc: :block_range | +| Explorer.Chain.Block.Reward | block_rewards | [asc: :address_hash, asc: :address_type, asc: :block_hash] | +| Explorer.Chain.Block.SecondDegreeRelation | block_second_degree_relations | [asc: :nephew_hash, asc: :uncle_hash] | +| Explorer.Chain.ContractMethod | contract_methods | [asc: :identified, asc: :abi] +| Explorer.Chain.InternalTransaction | internal_transactions | [asc: :transaction_hash, asc: :index] | +| Explorer.Chain.Log | logs | [asc: :transaction_hash, asc: :index] | +| Explorer.Chain.StakingPool | staking_pools | :staking_address_hash | +| Explorer.Chain.StakingPoolsDelegator | staking_pools_delegators | [asc: :delegator_address_hash, asc: :pool_address_hash] | +| Explorer.Chain.Token | tokens | asc: :contract_address_hash | +| Explorer.Chain.TokenTransfer | token_transfers | [asc: :transaction_hash, asc: :log_index]| +| Explorer.Chain.Transaction | transactions | asc: :hash | +| Explorer.Chain.Transaction.Fork | transaction_forks | [asc: :uncle_hash, asc: :index] | +| Explorer.Market.MarketHistory | market_history | asc: :date | From 810dc48a2c7f236c7a4ab48e317b4ac26946a2bc Mon Sep 17 00:00:00 2001 From: pasqu4le Date: Tue, 10 Sep 2019 17:26:04 +0200 Subject: [PATCH 44/67] Enforce DB transaction's order between tables to prevent deadlocks --- apps/explorer/lib/explorer/chain.ex | 51 +- apps/explorer/lib/explorer/chain/import.ex | 4 +- .../runner/address/current_token_balances.ex | 6 + .../explorer/chain/import/runner/blocks.ex | 511 +++++++++--------- .../import/runner/internal_transactions.ex | 29 +- .../chain/import/runner/staking_pools.ex | 42 +- .../explorer/chain/import/runner/tokens.ex | 40 +- .../chain/import/runner/transactions.ex | 44 +- .../lib/explorer/chain/import/stage.ex | 20 + .../chain/import/stage/address_referencing.ex | 28 +- .../chain/import/stage/block_following.ex | 30 + .../chain/import/stage/block_referencing.ex | 30 + .../chain/import/runner/blocks_test.exs | 43 +- docs/sharelocks.md | 75 ++- 14 files changed, 542 insertions(+), 411 deletions(-) create mode 100644 apps/explorer/lib/explorer/chain/import/stage/block_following.ex create mode 100644 apps/explorer/lib/explorer/chain/import/stage/block_referencing.ex diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index 9b70373228..6de1adc9ff 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -595,12 +595,15 @@ defmodule Explorer.Chain do def create_decompiled_smart_contract(attrs) do changeset = DecompiledSmartContract.changeset(%DecompiledSmartContract{}, attrs) + # Enforce ShareLocks tables order (see docs: sharelocks.md) Multi.new() + |> Multi.run(:set_address_decompiled, fn repo, _ -> + set_address_decompiled(repo, Changeset.get_field(changeset, :address_hash)) + end) |> Multi.insert(:decompiled_smart_contract, changeset, on_conflict: :replace_all, conflict_target: [:decompiler_version, :address_hash] ) - |> Multi.run(:set_address_decompiled, &set_address_decompiled/2) |> Repo.transaction() |> case do {:ok, %{decompiled_smart_contract: decompiled_smart_contract}} -> {:ok, decompiled_smart_contract} @@ -2519,12 +2522,18 @@ defmodule Explorer.Chain do |> SmartContract.changeset(attrs) |> Changeset.put_change(:external_libraries, external_libraries) + address_hash = Changeset.get_field(smart_contract_changeset, :address_hash) + + # Enforce ShareLocks tables order (see docs: sharelocks.md) insert_result = Multi.new() + |> Multi.run(:set_address_verified, fn repo, _ -> set_address_verified(repo, address_hash) end) + |> Multi.run(:clear_primary_address_names, fn repo, _ -> clear_primary_address_names(repo, address_hash) end) + |> Multi.run(:insert_address_name, fn repo, _ -> + name = Changeset.get_field(smart_contract_changeset, :name) + create_address_name(repo, name, address_hash) + end) |> Multi.insert(:smart_contract, smart_contract_changeset) - |> Multi.run(:clear_primary_address_names, &clear_primary_address_names/2) - |> Multi.run(:insert_address_name, &create_address_name/2) - |> Multi.run(:set_address_verified, &set_address_verified/2) |> Repo.transaction() case insert_result do @@ -2539,7 +2548,7 @@ defmodule Explorer.Chain do end end - defp set_address_verified(repo, %{smart_contract: %SmartContract{address_hash: address_hash}}) do + defp set_address_verified(repo, address_hash) do query = from( address in Address, @@ -2552,7 +2561,7 @@ defmodule Explorer.Chain do end end - defp set_address_decompiled(repo, %{decompiled_smart_contract: %DecompiledSmartContract{address_hash: address_hash}}) do + defp set_address_decompiled(repo, address_hash) do query = from( address in Address, @@ -2561,24 +2570,29 @@ defmodule Explorer.Chain do case repo.update_all(query, set: [decompiled: true]) do {1, _} -> {:ok, []} - _ -> {:error, "There was an error annotating that the address has been verified."} + _ -> {:error, "There was an error annotating that the address has been decompiled."} end end - defp clear_primary_address_names(repo, %{smart_contract: %SmartContract{address_hash: address_hash}}) do - clear_primary_query = + defp clear_primary_address_names(repo, address_hash) do + query = from( address_name in Address.Name, where: address_name.address_hash == ^address_hash, - update: [set: [primary: false]] + # Enforce Name ShareLocks order (see docs: sharelocks.md) + order_by: [asc: :address_hash, asc: :name], + lock: "FOR UPDATE" ) - repo.update_all(clear_primary_query, []) + repo.update_all( + from(n in Address.Name, join: s in subquery(query), on: n.address_hash == s.address_hash), + set: [primary: false] + ) {:ok, []} end - defp create_address_name(repo, %{smart_contract: %SmartContract{name: name, address_hash: address_hash}}) do + defp create_address_name(repo, name, address_hash) do params = %{ address_hash: address_hash, name: name, @@ -3016,8 +3030,15 @@ defmodule Explorer.Chain do address_name_opts = [on_conflict: :nothing, conflict_target: [:address_hash, :name]] + # Enforce ShareLocks tables order (see docs: sharelocks.md) insert_result = Multi.new() + |> Multi.run( + :address_name, + fn repo, _ -> + {:ok, repo.insert(address_name_changeset, address_name_opts)} + end + ) |> Multi.run(:token, fn repo, _ -> with {:error, %Changeset{errors: [{^stale_error_field, {^stale_error_message, []}}]}} <- repo.insert(token_changeset, token_opts) do @@ -3025,12 +3046,6 @@ defmodule Explorer.Chain do {:ok, token} end end) - |> Multi.run( - :address_name, - fn repo, _ -> - {:ok, repo.insert(address_name_changeset, address_name_opts)} - end - ) |> Repo.transaction() case insert_result do diff --git a/apps/explorer/lib/explorer/chain/import.ex b/apps/explorer/lib/explorer/chain/import.ex index 2e647a873d..b682e1adf1 100644 --- a/apps/explorer/lib/explorer/chain/import.ex +++ b/apps/explorer/lib/explorer/chain/import.ex @@ -10,7 +10,9 @@ defmodule Explorer.Chain.Import do @stages [ Import.Stage.Addresses, - Import.Stage.AddressReferencing + Import.Stage.AddressReferencing, + Import.Stage.BlockReferencing, + Import.Stage.BlockFollowing ] # in order so that foreign keys are inserted before being referenced diff --git a/apps/explorer/lib/explorer/chain/import/runner/address/current_token_balances.ex b/apps/explorer/lib/explorer/chain/import/runner/address/current_token_balances.ex index 503d5c877b..8e46986127 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/address/current_token_balances.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/address/current_token_balances.ex @@ -106,7 +106,12 @@ defmodule Explorer.Chain.Import.Runner.Address.CurrentTokenBalances do |> Map.put_new(:timeout, @timeout) |> Map.put(:timestamps, timestamps) + # Enforce ShareLocks tables order (see docs: sharelocks.md) multi + |> Multi.run(:acquire_contract_address_tokens, fn repo, _ -> + contract_address_hashes = changes_list |> Enum.map(& &1.token_contract_address_hash) |> Enum.uniq() + Tokens.acquire_contract_address_tokens(repo, contract_address_hashes) + end) |> Multi.run(:address_current_token_balances, fn repo, _ -> insert(repo, changes_list, insert_options) end) @@ -117,6 +122,7 @@ defmodule Explorer.Chain.Import.Runner.Address.CurrentTokenBalances do } -> token_holder_count_deltas = upserted_balances_to_holder_count_deltas(upserted_balances) + # ShareLocks order already enforced by `acquire_contract_address_tokens` (see docs: sharelocks.md) Tokens.update_holder_counts_with_deltas( repo, token_holder_count_deltas, diff --git a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex index 58cfb67ce5..f893df7684 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex @@ -5,10 +5,10 @@ defmodule Explorer.Chain.Import.Runner.Blocks do require Ecto.Query - import Ecto.Query, only: [from: 2, lock: 2, order_by: 2, subquery: 1] + import Ecto.Query, only: [from: 2, subquery: 1] alias Ecto.{Changeset, Multi, Repo} - alias Explorer.Chain.{Address, Block, Hash, Import, InternalTransaction, Log, TokenTransfer, Transaction} + alias Explorer.Chain.{Address, Block, Import, InternalTransaction, Log, TokenTransfer, Transaction} alias Explorer.Chain.Block.Reward alias Explorer.Chain.Import.Runner alias Explorer.Chain.Import.Runner.Address.CurrentTokenBalances @@ -44,58 +44,100 @@ defmodule Explorer.Chain.Import.Runner.Blocks do |> Map.put_new(:timeout, @timeout) |> Map.put(:timestamps, timestamps) - ordered_consensus_block_numbers = ordered_consensus_block_numbers(changes_list) + hashes = Enum.map(changes_list, & &1.hash) + consensus_block_numbers = consensus_block_numbers(changes_list) where_invalid_neighbour = where_invalid_neighbour(changes_list) - where_forked = where_forked(changes_list) + # Enforce ShareLocks tables order (see docs: sharelocks.md) multi - |> Multi.run(:derive_transaction_forks, fn repo, _ -> - derive_transaction_forks(%{ - repo: repo, - timeout: options[Runner.Transaction.Forks.option_key()][:timeout] || Runner.Transaction.Forks.timeout(), - timestamps: timestamps, - where_forked: where_forked - }) + |> Multi.run(:acquire_blocks, fn repo, _ -> + acquire_blocks(repo, hashes, consensus_block_numbers, where_invalid_neighbour) end) |> Multi.run(:lose_consensus, fn repo, _ -> - lose_consensus(repo, ordered_consensus_block_numbers, insert_options) + lose_consensus(repo, consensus_block_numbers, insert_options) end) |> Multi.run(:lose_invalid_neighbour_consensus, fn repo, _ -> lose_invalid_neighbour_consensus(repo, where_invalid_neighbour, insert_options) end) - |> Multi.run(:remove_nonconsensus_data, fn repo, - %{ - lose_consensus: lost_consensus_blocks, - lose_invalid_neighbour_consensus: lost_consensus_neighbours - } -> + |> Multi.run(:nonconsensus_block_numbers, fn _repo, + %{ + lose_consensus: lost_consensus_blocks, + lose_invalid_neighbour_consensus: lost_consensus_neighbours + } -> nonconsensus_block_numbers = (lost_consensus_blocks ++ lost_consensus_neighbours) - |> Enum.map(fn %{number: number} -> - number - end) |> Enum.sort() |> Enum.dedup() - remove_nonconsensus_data( - repo, - nonconsensus_block_numbers, - insert_options - ) + {:ok, nonconsensus_block_numbers} + end) + |> Multi.run(:blocks, fn repo, _ -> + insert(repo, changes_list, insert_options) + end) + |> Multi.run(:uncle_fetched_block_second_degree_relations, fn repo, %{blocks: blocks} when is_list(blocks) -> + update_block_second_degree_relations(repo, hashes, %{ + timeout: + options[Runner.Block.SecondDegreeRelations.option_key()][:timeout] || + Runner.Block.SecondDegreeRelations.timeout(), + timestamps: timestamps + }) + end) + |> Multi.run(:delete_rewards, fn repo, _ -> + delete_rewards(repo, changes_list, insert_options) end) - # MUST be after `:derive_transaction_forks`, which depends on values in `transactions` table |> Multi.run(:fork_transactions, fn repo, _ -> fork_transactions(%{ repo: repo, timeout: options[Runner.Transactions.option_key()][:timeout] || Runner.Transactions.timeout(), timestamps: timestamps, - where_forked: where_forked + blocks_changes: changes_list + }) + end) + |> Multi.run(:derive_transaction_forks, fn repo, %{fork_transactions: transactions} -> + derive_transaction_forks(%{ + repo: repo, + timeout: options[Runner.Transaction.Forks.option_key()][:timeout] || Runner.Transaction.Forks.timeout(), + timestamps: timestamps, + transactions: transactions }) end) + |> Multi.run(:remove_nonconsensus_logs, fn repo, + %{ + nonconsensus_block_numbers: nonconsensus_block_numbers, + fork_transactions: transactions + } -> + remove_nonconsensus_logs(repo, nonconsensus_block_numbers, transactions, insert_options) + end) + |> Multi.run(:acquire_internal_transactions, fn repo, + %{ + nonconsensus_block_numbers: nonconsensus_block_numbers, + fork_transactions: transactions + } -> + acquire_internal_transactions(repo, nonconsensus_block_numbers, hashes, transactions) + end) + |> Multi.run(:remove_nonconsensus_internal_transactions, fn repo, + %{ + nonconsensus_block_numbers: + nonconsensus_block_numbers, + fork_transactions: transactions + } -> + remove_nonconsensus_internal_transactions(repo, nonconsensus_block_numbers, transactions, insert_options) + end) + |> Multi.run(:internal_transaction_transaction_block_number, fn repo, _ -> + update_internal_transaction_block_number(repo, hashes) + end) + |> Multi.run(:acquire_contract_address_tokens, fn repo, _ -> + acquire_contract_address_tokens(repo, consensus_block_numbers) + end) + |> Multi.run(:remove_nonconsensus_token_transfers, fn repo, + %{nonconsensus_block_numbers: nonconsensus_block_numbers} -> + remove_nonconsensus_token_transfers(repo, nonconsensus_block_numbers, insert_options) + end) |> Multi.run(:delete_address_token_balances, fn repo, _ -> - delete_address_token_balances(repo, ordered_consensus_block_numbers, insert_options) + delete_address_token_balances(repo, consensus_block_numbers, insert_options) end) |> Multi.run(:delete_address_current_token_balances, fn repo, _ -> - delete_address_current_token_balances(repo, ordered_consensus_block_numbers, insert_options) + delete_address_current_token_balances(repo, consensus_block_numbers, insert_options) end) |> Multi.run(:derive_address_current_token_balances, fn repo, %{ @@ -112,99 +154,89 @@ defmodule Explorer.Chain.Import.Runner.Blocks do deltas = CurrentTokenBalances.token_holder_count_deltas(%{deleted: deleted, inserted: inserted}) Tokens.update_holder_counts_with_deltas(repo, deltas, insert_options) end) - |> Multi.run(:delete_rewards, fn repo, _ -> - delete_rewards(repo, changes_list, insert_options) - end) - |> Multi.run(:blocks, fn repo, _ -> - insert(repo, changes_list, insert_options) - end) - |> Multi.run(:uncle_fetched_block_second_degree_relations, fn repo, %{blocks: blocks} when is_list(blocks) -> - update_block_second_degree_relations( - repo, - blocks, - %{ - timeout: - options[Runner.Block.SecondDegreeRelations.option_key()][:timeout] || - Runner.Block.SecondDegreeRelations.timeout(), - timestamps: timestamps - } - ) - end) - |> Multi.run(:internal_transaction_transaction_block_number, fn repo, %{blocks: blocks} when is_list(blocks) -> - update_internal_transaction_block_number(repo, blocks) - end) end @impl Runner def timeout, do: @timeout - defp derive_transaction_forks(%{ - repo: repo, - timeout: timeout, - timestamps: %{inserted_at: inserted_at, updated_at: updated_at}, - where_forked: where_forked - }) do - multi = - Multi.new() - |> Multi.run(:get_forks, fn repo, _ -> - query = - from(transaction in where_forked, - select: %{ - uncle_hash: transaction.block_hash, - index: transaction.index, - hash: transaction.hash, - inserted_at: type(^inserted_at, transaction.inserted_at), - updated_at: type(^updated_at, transaction.updated_at) - } - ) - - transactions = repo.all(query) - {:ok, transactions} - end) - |> Multi.run(:insert_transaction_forks, fn repo, %{get_forks: transactions} -> - # Enforce Fork ShareLocks order (see docs: sharelocks.md) - ordered_forks = Enum.sort_by(transactions, &{&1.uncle_hash, &1.index}) - - {_total, result} = - repo.insert_all( - Transaction.Fork, - ordered_forks, - conflict_target: [:uncle_hash, :index], - on_conflict: - from( - transaction_fork in Transaction.Fork, - update: [set: [hash: fragment("EXCLUDED.hash")]], - where: fragment("EXCLUDED.hash <> ?", transaction_fork.hash) - ), - returning: [:uncle_hash, :hash] - ) - - {:ok, result} - end) + defp acquire_blocks(repo, hashes, consensus_block_numbers, where_invalid_neighbour) do + query = + from( + block in where_invalid_neighbour, + or_where: block.number in ^consensus_block_numbers, + or_where: block.hash in ^hashes, + select: block.hash, + # Enforce Block ShareLocks order (see docs: sharelocks.md) + order_by: [asc: block.hash], + lock: "FOR UPDATE" + ) + + blocks = repo.all(query) + {:ok, blocks} + end - with {:ok, %{insert_transaction_forks: rows}} <- repo.transaction(multi, timeout: timeout) do - derived_transaction_forks = Enum.map(rows, &Map.take(&1, [:uncle_hash, :hash])) + defp acquire_contract_address_tokens(repo, consensus_block_numbers) do + query = + from(address_current_token_balance in Address.CurrentTokenBalance, + where: address_current_token_balance.block_number in ^consensus_block_numbers, + select: address_current_token_balance.token_contract_address_hash + ) - {:ok, derived_transaction_forks} - end + contract_address_hashes = repo.all(query) + + Tokens.acquire_contract_address_tokens(repo, contract_address_hashes) + end + + defp acquire_internal_transactions(repo, nonconsensus_block_numbers, hashes, forked_transactions) do + forked_transaction_hashes = Enum.map(forked_transactions, & &1.hash) + + query = + from(internal_transaction in InternalTransaction, + join: transaction in Transaction, + on: internal_transaction.transaction_hash == transaction.hash, + where: transaction.block_number in ^nonconsensus_block_numbers, + or_where: transaction.block_hash in ^hashes, + or_where: transaction.hash in ^forked_transaction_hashes, + select: {internal_transaction.transaction_hash, internal_transaction.index}, + # Enforce InternalTransaction ShareLocks order (see docs: sharelocks.md) + order_by: [ + internal_transaction.transaction_hash, + internal_transaction.index + ], + # NOTE: find a better way to know the alias that ecto gives to token + lock: "FOR UPDATE OF i0" + ) + + {:ok, repo.all(query)} end defp fork_transactions(%{ repo: repo, timeout: timeout, timestamps: %{updated_at: updated_at}, - where_forked: where_forked + blocks_changes: blocks_changes }) do query = - where_forked - # Enforce Transaction ShareLocks order (see docs: sharelocks.md) - |> order_by(asc: :hash) - |> lock("FOR UPDATE") + from( + transaction in where_forked(blocks_changes), + select: %{ + block_hash: transaction.block_hash, + index: transaction.index, + hash: transaction.hash + }, + # Enforce Transaction ShareLocks order (see docs: sharelocks.md) + order_by: [asc: :hash], + lock: "FOR UPDATE" + ) + + transactions = repo.all(query) + + hashes = Enum.map(transactions, & &1.hash) update_query = - from(t in Transaction, - join: s in subquery(query), - on: t.hash == s.hash, + from( + t in Transaction, + where: t.hash in ^hashes, update: [ set: [ block_hash: nil, @@ -222,15 +254,53 @@ defmodule Explorer.Chain.Import.Runner.Blocks do ) try do - {_, result} = repo.update_all(update_query, [], timeout: timeout) + {_num, _res} = repo.update_all(update_query, [], timeout: timeout) - {:ok, result} + {:ok, transactions} rescue postgrex_error in Postgrex.Error -> {:error, %{exception: postgrex_error}} end end + defp derive_transaction_forks(%{ + repo: repo, + timeout: timeout, + timestamps: %{inserted_at: inserted_at, updated_at: updated_at}, + transactions: transactions + }) do + transaction_forks = + transactions + |> Enum.map(fn transaction -> + %{ + uncle_hash: transaction.block_hash, + index: transaction.index, + hash: transaction.hash, + inserted_at: inserted_at, + updated_at: updated_at + } + end) + # Enforce Fork ShareLocks order (see docs: sharelocks.md) + |> Enum.sort_by(&{&1.uncle_hash, &1.index}) + + {_total, result} = + repo.insert_all( + Transaction.Fork, + transaction_forks, + conflict_target: [:uncle_hash, :index], + on_conflict: + from( + transaction_fork in Transaction.Fork, + update: [set: [hash: fragment("EXCLUDED.hash")]], + where: fragment("EXCLUDED.hash <> ?", transaction_fork.hash) + ), + returning: [:uncle_hash, :hash], + timeout: timeout + ) + + {:ok, result} + end + @spec insert(Repo.t(), [map()], %{ optional(:on_conflict) => Runner.on_conflict(), required(:timeout) => timeout, @@ -288,90 +358,46 @@ defmodule Explorer.Chain.Import.Runner.Blocks do ) end - defp ordered_consensus_block_numbers(blocks_changes) when is_list(blocks_changes) do + defp consensus_block_numbers(blocks_changes) when is_list(blocks_changes) do blocks_changes - |> Enum.reduce(MapSet.new(), fn - %{consensus: true, number: number}, acc -> - MapSet.put(acc, number) - - %{consensus: false}, acc -> - acc - end) - |> Enum.sort() + |> Enum.filter(& &1.consensus) + |> Enum.map(& &1.number) end defp lose_consensus(_, [], _), do: {:ok, []} - defp lose_consensus(repo, ordered_consensus_block_number, %{timeout: timeout, timestamps: %{updated_at: updated_at}}) - when is_list(ordered_consensus_block_number) do - query = - from( - block in Block, - where: block.number in ^ordered_consensus_block_number, - # Enforce Block ShareLocks order (see docs: sharelocks.md) - order_by: [asc: block.hash], - lock: "FOR UPDATE" + defp lose_consensus(repo, consensus_block_number, %{timeout: timeout, timestamps: %{updated_at: updated_at}}) + when is_list(consensus_block_number) do + # ShareLocks order already enforced by `acquire_blocks` (see docs: sharelocks.md) + {_, result} = + repo.update_all( + from(block in Block, where: block.number in ^consensus_block_number, select: block.number), + [set: [consensus: false, updated_at: updated_at]], + timeout: timeout ) - try do - {_, result} = - repo.update_all( - from(b in Block, join: s in subquery(query), on: b.hash == s.hash, select: [:hash, :number]), - [set: [consensus: false, updated_at: updated_at]], - timeout: timeout - ) - - {:ok, result} - rescue - postgrex_error in Postgrex.Error -> - {:error, %{exception: postgrex_error, consensus_block_numbers: ordered_consensus_block_number}} - end + {:ok, result} + rescue + postgrex_error in Postgrex.Error -> + {:error, %{exception: postgrex_error, consensus_block_numbers: consensus_block_number}} end defp lose_invalid_neighbour_consensus(repo, where_invalid_neighbour, %{ timeout: timeout, timestamps: %{updated_at: updated_at} }) do - query = - from( - block in where_invalid_neighbour, - # Enforce Block ShareLocks order (see docs: sharelocks.md) - order_by: [asc: block.hash], - lock: "FOR UPDATE" + # ShareLocks order already enforced by `acquire_blocks` (see docs: sharelocks.md) + {_, result} = + repo.update_all( + from(block in where_invalid_neighbour, select: block.number), + [set: [consensus: false, updated_at: updated_at]], + timeout: timeout ) - try do - {_, result} = - repo.update_all( - from(b in Block, join: s in subquery(query), on: b.hash == s.hash, select: [:hash, :number]), - [set: [consensus: false, updated_at: updated_at]], - timeout: timeout - ) - - {:ok, result} - rescue - postgrex_error in Postgrex.Error -> - {:error, %{exception: postgrex_error, where_invalid_neighbour: where_invalid_neighbour}} - end - end - - defp remove_nonconsensus_data( - repo, - nonconsensus_block_numbers, - insert_options - ) do - with {:ok, deleted_token_transfers} <- - remove_nonconsensus_token_transfers(repo, nonconsensus_block_numbers, insert_options), - {:ok, deleted_logs} <- remove_nonconsensus_logs(repo, nonconsensus_block_numbers, insert_options), - {:ok, deleted_internal_transactions} <- - remove_nonconsensus_internal_transactions(repo, nonconsensus_block_numbers, insert_options) do - {:ok, - %{ - token_transfers: deleted_token_transfers, - logs: deleted_logs, - internal_transactions: deleted_internal_transactions - }} - end + {:ok, result} + rescue + postgrex_error in Postgrex.Error -> + {:error, %{exception: postgrex_error, where_invalid_neighbour: where_invalid_neighbour}} end defp remove_nonconsensus_token_transfers(repo, nonconsensus_block_numbers, %{timeout: timeout}) do @@ -407,37 +433,27 @@ defmodule Explorer.Chain.Import.Runner.Blocks do end end - defp remove_nonconsensus_internal_transactions(repo, nonconsensus_block_numbers, %{timeout: timeout}) do + defp remove_nonconsensus_internal_transactions(repo, nonconsensus_block_numbers, forked_transactions, %{ + timeout: timeout + }) do + forked_transaction_hashes = Enum.map(forked_transactions, & &1.hash) + transaction_query = from(transaction in Transaction, where: transaction.block_number in ^nonconsensus_block_numbers, - select: map(transaction, [:hash]), - order_by: transaction.hash + or_where: transaction.hash in ^forked_transaction_hashes, + select: map(transaction, [:hash]) ) - ordered_internal_transactions = + query = from(internal_transaction in InternalTransaction, inner_join: transaction in subquery(transaction_query), on: internal_transaction.transaction_hash == transaction.hash, - select: map(internal_transaction, [:transaction_hash, :index]), - # Enforce InternalTransaction ShareLocks order (see docs: sharelocks.md) - order_by: [ - internal_transaction.transaction_hash, - internal_transaction.index - ], - lock: "FOR UPDATE OF i0" - ) - - query = - from(internal_transaction in InternalTransaction, - select: map(internal_transaction, [:transaction_hash, :index]), - inner_join: ordered_internal_transaction in subquery(ordered_internal_transactions), - on: - ordered_internal_transaction.transaction_hash == internal_transaction.transaction_hash and - ordered_internal_transaction.index == internal_transaction.index + select: map(internal_transaction, [:transaction_hash, :index]) ) try do + # ShareLocks order already enforced by `acquire_internal_transactions` (see docs: sharelocks.md) {_count, deleted_internal_transactions} = repo.delete_all(query, timeout: timeout) {:ok, deleted_internal_transactions} @@ -447,10 +463,13 @@ defmodule Explorer.Chain.Import.Runner.Blocks do end end - defp remove_nonconsensus_logs(repo, nonconsensus_block_numbers, %{timeout: timeout}) do + defp remove_nonconsensus_logs(repo, nonconsensus_block_numbers, forked_transactions, %{timeout: timeout}) do + forked_transaction_hashes = Enum.map(forked_transactions, & &1.hash) + transaction_query = from(transaction in Transaction, where: transaction.block_number in ^nonconsensus_block_numbers, + or_where: transaction.hash in ^forked_transaction_hashes, select: map(transaction, [:hash]), order_by: transaction.hash ) @@ -487,10 +506,10 @@ defmodule Explorer.Chain.Import.Runner.Blocks do defp delete_address_token_balances(_, [], _), do: {:ok, []} - defp delete_address_token_balances(repo, ordered_consensus_block_numbers, %{timeout: timeout}) do + defp delete_address_token_balances(repo, consensus_block_numbers, %{timeout: timeout}) do ordered_query = from(address_token_balance in Address.TokenBalance, - where: address_token_balance.block_number in ^ordered_consensus_block_numbers, + where: address_token_balance.block_number in ^consensus_block_numbers, select: map(address_token_balance, [:address_hash, :token_contract_address_hash, :block_number]), # Enforce TokenBalance ShareLocks order (see docs: sharelocks.md) order_by: [ @@ -518,16 +537,16 @@ defmodule Explorer.Chain.Import.Runner.Blocks do {:ok, deleted_address_token_balances} rescue postgrex_error in Postgrex.Error -> - {:error, %{exception: postgrex_error, block_numbers: ordered_consensus_block_numbers}} + {:error, %{exception: postgrex_error, block_numbers: consensus_block_numbers}} end end defp delete_address_current_token_balances(_, [], _), do: {:ok, []} - defp delete_address_current_token_balances(repo, ordered_consensus_block_numbers, %{timeout: timeout}) do + defp delete_address_current_token_balances(repo, consensus_block_numbers, %{timeout: timeout}) do ordered_query = from(address_current_token_balance in Address.CurrentTokenBalance, - where: address_current_token_balance.block_number in ^ordered_consensus_block_numbers, + where: address_current_token_balance.block_number in ^consensus_block_numbers, select: map(address_current_token_balance, [:address_hash, :token_contract_address_hash]), # Enforce CurrentTokenBalance ShareLocks order (see docs: sharelocks.md) order_by: [ @@ -561,7 +580,7 @@ defmodule Explorer.Chain.Import.Runner.Blocks do {:ok, deleted_address_current_token_balances} rescue postgrex_error in Postgrex.Error -> - {:error, %{exception: postgrex_error, block_numbers: ordered_consensus_block_numbers}} + {:error, %{exception: postgrex_error, block_numbers: consensus_block_numbers}} end end @@ -613,41 +632,26 @@ defmodule Explorer.Chain.Import.Runner.Blocks do ] ) - multi = - Multi.new() - |> Multi.run(:new_current_token_balance, fn repo, _ -> - new_current_token_balances = repo.all(new_current_token_balance_query) - {:ok, new_current_token_balances} - end) - |> Multi.run( - :insert_new_current_token_balance, - fn repo, %{new_current_token_balance: new_current_token_balance} -> - # Enforce CurrentTokenBalance ShareLocks order (see docs: sharelocks.md) - ordered_current_token_balance = - Enum.sort_by( - new_current_token_balance, - &{&1.address_hash, &1.token_contract_address_hash} - ) - - {_total, result} = - repo.insert_all( - Address.CurrentTokenBalance, - ordered_current_token_balance, - # No `ON CONFLICT` because `delete_address_current_token_balances` - # should have removed any conflicts. - returning: [:address_hash, :token_contract_address_hash, :block_number, :value] - ) - - {:ok, result} - end + ordered_current_token_balance = + new_current_token_balance_query + |> repo.all() + # Enforce CurrentTokenBalance ShareLocks order (see docs: sharelocks.md) + |> Enum.sort_by(&{&1.address_hash, &1.token_contract_address_hash}) + + {_total, result} = + repo.insert_all( + Address.CurrentTokenBalance, + ordered_current_token_balance, + # No `ON CONFLICT` because `delete_address_current_token_balances` + # should have removed any conflicts. + returning: [:address_hash, :token_contract_address_hash, :block_number, :value], + timeout: timeout ) - with {:ok, %{insert_new_current_token_balance: rows}} <- repo.transaction(multi, timeout: timeout) do - derived_address_current_token_balances = - Enum.map(rows, &Map.take(&1, [:address_hash, :token_contract_address_hash, :block_number, :value])) + derived_address_current_token_balances = + Enum.map(result, &Map.take(&1, [:address_hash, :token_contract_address_hash, :block_number, :value])) - {:ok, derived_address_current_token_balances} - end + {:ok, derived_address_current_token_balances} end # `block_rewards` are linked to `blocks.hash`, but fetched by `blocks.number`, so when a block with the same number is @@ -691,13 +695,11 @@ defmodule Explorer.Chain.Import.Runner.Blocks do end end - defp update_block_second_degree_relations(repo, blocks, %{timeout: timeout, timestamps: %{updated_at: updated_at}}) - when is_list(blocks) do - uncle_hashes = - blocks - |> MapSet.new(& &1.hash) - |> MapSet.to_list() - + defp update_block_second_degree_relations(repo, uncle_hashes, %{ + timeout: timeout, + timestamps: %{updated_at: updated_at} + }) + when is_list(uncle_hashes) do query = from( bsdr in Block.SecondDegreeRelation, @@ -725,9 +727,7 @@ defmodule Explorer.Chain.Import.Runner.Blocks do end end - defp update_internal_transaction_block_number(repo, blocks) when is_list(blocks) do - blocks_hashes = Enum.map(blocks, & &1.hash) - + defp update_internal_transaction_block_number(repo, blocks_hashes) when is_list(blocks_hashes) do query = from( internal_transaction in InternalTransaction, @@ -736,26 +736,11 @@ defmodule Explorer.Chain.Import.Runner.Blocks do join: block in Block, on: block.hash == transaction.block_hash, where: block.hash in ^blocks_hashes, - select: %{ - transaction_hash: internal_transaction.transaction_hash, - index: internal_transaction.index, - block_number: block.number - }, - # Enforce InternalTransaction ShareLocks order (see docs: sharelocks.md) - order_by: [asc: :transaction_hash, asc: :index], - # NOTE: find a better way to know the alias that ecto gives to internal_transaction - lock: "FOR UPDATE OF i0" - ) - - update_query = - from( - i in InternalTransaction, - join: s in subquery(query), - on: i.transaction_hash == s.transaction_hash and i.index == s.index, - update: [set: [block_number: s.block_number]] + update: [set: [block_number: block.number]] ) - {total, _} = repo.update_all(update_query, []) + # ShareLocks order already enforced by `acquire_internal_transactions` (see docs: sharelocks.md) + {total, _} = repo.update_all(query, []) {:ok, total} end diff --git a/apps/explorer/lib/explorer/chain/import/runner/internal_transactions.ex b/apps/explorer/lib/explorer/chain/import/runner/internal_transactions.ex index c7422f074b..93e3bdb6b5 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/internal_transactions.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/internal_transactions.ex @@ -47,12 +47,16 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactions do update_transactions_options = %{timeout: transactions_timeout, timestamps: timestamps} + # Enforce ShareLocks tables order (see docs: sharelocks.md) multi + |> Multi.run(:acquire_transactions, fn repo, _ -> + acquire_transactions(repo, changes_list) + end) |> Multi.run(:internal_transactions, fn repo, _ -> insert(repo, changes_list, insert_options) end) - |> Multi.run(:internal_transactions_indexed_at_transactions, fn repo, _ -> - update_transactions(repo, changes_list, update_transactions_options) + |> Multi.run(:internal_transactions_indexed_at_transactions, fn repo, %{acquire_transactions: transaction_hashes} -> + update_transactions(repo, transaction_hashes, update_transactions_options) end) end @@ -144,11 +148,7 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactions do ) end - defp update_transactions(repo, internal_transactions, %{ - timeout: timeout, - timestamps: timestamps - }) - when is_list(internal_transactions) do + defp acquire_transactions(repo, internal_transactions) do transaction_hashes = internal_transactions |> MapSet.new(& &1.transaction_hash) @@ -159,16 +159,27 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactions do t in Transaction, where: t.hash in ^transaction_hashes, where: not is_nil(t.block_hash), + select: t.hash, # Enforce Transaction ShareLocks order (see docs: sharelocks.md) order_by: t.hash, lock: "FOR UPDATE" ) + hashes = repo.all(query) + + {:ok, hashes} + end + + defp update_transactions(repo, transaction_hashes, %{ + timeout: timeout, + timestamps: timestamps + }) + when is_list(transaction_hashes) do update_query = from( t in Transaction, - join: s in subquery(query), - on: t.hash == s.hash, + where: t.hash in ^transaction_hashes, + # ShareLocks order already enforced by `acquire_transactions` (see docs: sharelocks.md) update: [ set: [ internal_transactions_indexed_at: ^timestamps.updated_at, diff --git a/apps/explorer/lib/explorer/chain/import/runner/staking_pools.ex b/apps/explorer/lib/explorer/chain/import/runner/staking_pools.ex index 13ba8cb5f9..fa4fa97a75 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/staking_pools.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/staking_pools.ex @@ -40,7 +40,11 @@ defmodule Explorer.Chain.Import.Runner.StakingPools do |> Map.put_new(:timeout, @timeout) |> Map.put(:timestamps, timestamps) + # Enforce ShareLocks tables order (see docs: sharelocks.md) multi + |> Multi.run(:acquire_all_staking_pools, fn repo, _ -> + acquire_all_staking_pools(repo) + end) |> Multi.run(:mark_as_deleted, fn repo, _ -> mark_as_deleted(repo, changes_list, insert_options) end) @@ -55,6 +59,20 @@ defmodule Explorer.Chain.Import.Runner.StakingPools do @impl Import.Runner def timeout, do: @timeout + defp acquire_all_staking_pools(repo) do + query = + from( + pool in StakingPool, + # Enforce StackingPool ShareLocks order (see docs: sharelocks.md) + order_by: pool.staking_address_hash, + lock: "FOR UPDATE" + ) + + pools = repo.all(query) + + {:ok, pools} + end + defp mark_as_deleted(repo, changes_list, %{timeout: timeout}) when is_list(changes_list) do addresses = Enum.map(changes_list, & &1.staking_address_hash) @@ -62,18 +80,12 @@ defmodule Explorer.Chain.Import.Runner.StakingPools do from( pool in StakingPool, where: pool.staking_address_hash not in ^addresses, - # Enforce StackingPool ShareLocks order (see docs: sharelocks.md) - order_by: pool.staking_address_hash, - lock: "FOR UPDATE" + # ShareLocks order already enforced by `acquire_all_staking_pools` (see docs: sharelocks.md) + update: [set: [is_deleted: true, is_active: false]] ) try do - {_, result} = - repo.update_all( - from(p in StakingPool, join: s in subquery(query), on: p.staking_address_hash == s.staking_address_hash), - [set: [is_deleted: true, is_active: false]], - timeout: timeout - ) + {_, result} = repo.update_all(query, [], timeout: timeout) {:ok, result} rescue @@ -143,19 +155,11 @@ defmodule Explorer.Chain.Import.Runner.StakingPools do total = repo.one!(total_query) if total > Decimal.new(0) do - query = + update_query = from( p in StakingPool, where: p.is_active == true, - # Enforce StackingPool ShareLocks order (see docs: sharelocks.md) - order_by: p.staking_address_hash, - lock: "FOR UPDATE" - ) - - update_query = - from(p in StakingPool, - join: s in subquery(query), - on: p.staking_address_hash == s.staking_address_hash, + # ShareLocks order already enforced by `acquire_all_staking_pools` (see docs: sharelocks.md) update: [ set: [ staked_ratio: p.staked_amount / ^total * 100, diff --git a/apps/explorer/lib/explorer/chain/import/runner/tokens.ex b/apps/explorer/lib/explorer/chain/import/runner/tokens.ex index 22621b022c..1062a8c8bb 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/tokens.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/tokens.ex @@ -21,10 +21,26 @@ defmodule Explorer.Chain.Import.Runner.Tokens do @type holder_count :: non_neg_integer() @type token_holder_count :: %{contract_address_hash: Hash.Address.t(), count: holder_count()} + def acquire_contract_address_tokens(repo, contract_address_hashes) do + token_query = + from( + token in Token, + where: token.contract_address_hash in ^contract_address_hashes, + # Enforce Token ShareLocks order (see docs: sharelocks.md) + order_by: token.contract_address_hash, + lock: "FOR UPDATE" + ) + + tokens = repo.all(token_query) + + {:ok, tokens} + end + def update_holder_counts_with_deltas(repo, token_holder_count_deltas, %{ timeout: timeout, timestamps: %{updated_at: updated_at} }) do + # NOTE that acquire_contract_address_tokens needs to be called before this {hashes, deltas} = token_holder_count_deltas |> Enum.map(fn %{contract_address_hash: contract_address_hash, delta: delta} -> @@ -43,35 +59,21 @@ defmodule Explorer.Chain.Import.Runner.Tokens do ^deltas ), on: token.contract_address_hash == deltas.contract_address_hash, - select: %{ - contract_address_hash: token.contract_address_hash, - delta: deltas.delta - }, where: not is_nil(token.holder_count), - # Enforce Token ShareLocks order (see docs: sharelocks.md) - order_by: token.contract_address_hash, - # NOTE: find a better way to know the alias that ecto gives to token - lock: "FOR UPDATE OF t0" - ) - - update_query = - from( - t in Token, - join: s in subquery(query), - on: t.contract_address_hash == s.contract_address_hash, + # ShareLocks order already enforced by `acquire_contract_address_tokens` (see docs: sharelocks.md) update: [ set: [ - holder_count: t.holder_count + s.delta, + holder_count: token.holder_count + deltas.delta, updated_at: ^updated_at ] ], select: %{ - contract_address_hash: t.contract_address_hash, - holder_count: t.holder_count + contract_address_hash: token.contract_address_hash, + holder_count: token.holder_count } ) - {_total, result} = repo.update_all(update_query, [], timeout: timeout) + {_total, result} = repo.update_all(query, [], timeout: timeout) {:ok, result} end diff --git a/apps/explorer/lib/explorer/chain/import/runner/transactions.ex b/apps/explorer/lib/explorer/chain/import/runner/transactions.ex index 89c89aae4d..5075ab5a09 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/transactions.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/transactions.ex @@ -42,13 +42,14 @@ defmodule Explorer.Chain.Import.Runner.Transactions do |> Map.put(:timestamps, timestamps) |> Map.put(:token_transfer_transaction_hash_set, token_transfer_transaction_hash_set(options)) + # Enforce ShareLocks tables order (see docs: sharelocks.md) multi + |> Multi.run(:recollated_transactions, fn repo, _ -> + discard_blocks_for_recollated_transactions(repo, changes_list, insert_options) + end) |> Multi.run(:transactions, fn repo, _ -> insert(repo, changes_list, insert_options) end) - |> Multi.run(:recollated_transactions, fn repo, %{transactions: transactions} -> - discard_blocks_for_recollated_transactions(repo, transactions, insert_options) - end) end @impl Import.Runner @@ -186,18 +187,39 @@ defmodule Explorer.Chain.Import.Runner.Transactions do defp put_internal_transactions_indexed_at?(_, _), do: false - defp discard_blocks_for_recollated_transactions(repo, transactions, %{ + defp discard_blocks_for_recollated_transactions(repo, changes_list, %{ timeout: timeout, timestamps: %{updated_at: updated_at} }) - when is_list(transactions) do - block_hashes = - transactions - |> Enum.filter(fn %{block_hash: block_hash, old_block_hash: old_block_hash} -> - not is_nil(old_block_hash) and block_hash != old_block_hash + when is_list(changes_list) do + {transactions_hashes, transactions_block_hashes} = + changes_list + |> Enum.filter(&Map.has_key?(&1, :block_hash)) + |> Enum.map(fn %{hash: hash, block_hash: block_hash} -> + {:ok, hash_bytes} = Hash.Full.dump(hash) + {:ok, block_hash_bytes} = Hash.Full.dump(block_hash) + {hash_bytes, block_hash_bytes} end) - |> MapSet.new(& &1.old_block_hash) - |> MapSet.to_list() + |> Enum.unzip() + + blocks_with_recollated_transactions = + from( + transaction in Transaction, + join: + new_transaction in fragment( + "(SELECT unnest(?::bytea[]) as hash, unnest(?::bytea[]) as block_hash)", + ^transactions_hashes, + ^transactions_block_hashes + ), + on: transaction.hash == new_transaction.hash, + where: transaction.block_hash != new_transaction.block_hash, + select: transaction.block_hash + ) + + block_hashes = + blocks_with_recollated_transactions + |> repo.all() + |> Enum.uniq() if Enum.empty?(block_hashes) do {:ok, []} diff --git a/apps/explorer/lib/explorer/chain/import/stage.ex b/apps/explorer/lib/explorer/chain/import/stage.ex index 6ad73d2568..dcd3da1cc1 100644 --- a/apps/explorer/lib/explorer/chain/import/stage.ex +++ b/apps/explorer/lib/explorer/chain/import/stage.ex @@ -47,4 +47,24 @@ defmodule Explorer.Chain.Import.Stage do runner.run(Multi.new(), changes_chunk, options) end) end + + @spec single_multi([Runner.t()], runner_to_changes_list, %{optional(atom()) => term()}) :: + {Multi.t(), runner_to_changes_list} + def single_multi(runners, runner_to_changes_list, options) do + runners + |> Enum.reduce({Multi.new(), runner_to_changes_list}, fn runner, {multi, remaining_runner_to_changes_list} -> + {changes_list, new_remaining_runner_to_changes_list} = Map.pop(remaining_runner_to_changes_list, runner) + + new_multi = + case changes_list do + nil -> + multi + + _ -> + runner.run(multi, changes_list, options) + end + + {new_multi, new_remaining_runner_to_changes_list} + end) + end end diff --git a/apps/explorer/lib/explorer/chain/import/stage/address_referencing.ex b/apps/explorer/lib/explorer/chain/import/stage/address_referencing.ex index b5037f3b04..2575a0eab8 100644 --- a/apps/explorer/lib/explorer/chain/import/stage/address_referencing.ex +++ b/apps/explorer/lib/explorer/chain/import/stage/address_referencing.ex @@ -4,7 +4,6 @@ defmodule Explorer.Chain.Import.Stage.AddressReferencing do `Explorer.Chain.Import.Stage.Addresses`. """ - alias Ecto.Multi alias Explorer.Chain.Import.{Runner, Stage} @behaviour Stage @@ -14,17 +13,6 @@ defmodule Explorer.Chain.Import.Stage.AddressReferencing do do: [ Runner.Address.CoinBalances, Runner.Blocks, - Runner.Block.Rewards, - Runner.Block.SecondDegreeRelations, - Runner.Transactions, - Runner.Transaction.Forks, - Runner.InternalTransactions, - Runner.InternalTransactionsIndexedAtBlocks, - Runner.Logs, - Runner.Tokens, - Runner.TokenTransfers, - Runner.Address.CurrentTokenBalances, - Runner.Address.TokenBalances, Runner.StakingPools, Runner.StakingPoolsDelegators ] @@ -32,21 +20,7 @@ defmodule Explorer.Chain.Import.Stage.AddressReferencing do @impl Stage def multis(runner_to_changes_list, options) do {final_multi, final_remaining_runner_to_changes_list} = - runners() - |> Enum.reduce({Multi.new(), runner_to_changes_list}, fn runner, {multi, remaining_runner_to_changes_list} -> - {changes_list, new_remaining_runner_to_changes_list} = Map.pop(remaining_runner_to_changes_list, runner) - - new_multi = - case changes_list do - nil -> - multi - - _ -> - runner.run(multi, changes_list, options) - end - - {new_multi, new_remaining_runner_to_changes_list} - end) + Stage.single_multi(runners(), runner_to_changes_list, options) {[final_multi], final_remaining_runner_to_changes_list} end diff --git a/apps/explorer/lib/explorer/chain/import/stage/block_following.ex b/apps/explorer/lib/explorer/chain/import/stage/block_following.ex new file mode 100644 index 0000000000..9b64fca993 --- /dev/null +++ b/apps/explorer/lib/explorer/chain/import/stage/block_following.ex @@ -0,0 +1,30 @@ +defmodule Explorer.Chain.Import.Stage.BlockFollowing do + @moduledoc """ + Imports any tables that follows and cannot be imported at the same time as + those imported by `Explorer.Chain.Import.Stage.Addresses`, + `Explorer.Chain.Import.Stage.AddressReferencing` and + `Explorer.Chain.Import.Stage.BlockReferencing` + """ + + alias Explorer.Chain.Import.{Runner, Stage} + + @behaviour Stage + + @impl Stage + def runners, + do: [ + Runner.InternalTransactionsIndexedAtBlocks, + Runner.Block.SecondDegreeRelations, + Runner.Block.Rewards, + Runner.InternalTransactions, + Runner.Address.CurrentTokenBalances + ] + + @impl Stage + def multis(runner_to_changes_list, options) do + {final_multi, final_remaining_runner_to_changes_list} = + Stage.single_multi(runners(), runner_to_changes_list, options) + + {[final_multi], final_remaining_runner_to_changes_list} + end +end diff --git a/apps/explorer/lib/explorer/chain/import/stage/block_referencing.ex b/apps/explorer/lib/explorer/chain/import/stage/block_referencing.ex new file mode 100644 index 0000000000..373ac7845b --- /dev/null +++ b/apps/explorer/lib/explorer/chain/import/stage/block_referencing.ex @@ -0,0 +1,30 @@ +defmodule Explorer.Chain.Import.Stage.BlockReferencing do + @moduledoc """ + Imports any tables that reference `t:Explorer.Chain.Block.t/0` and that were + imported by `Explorer.Chain.Import.Stage.Addresses` and + `Explorer.Chain.Import.Stage.AddressReferencing`. + """ + + alias Explorer.Chain.Import.{Runner, Stage} + + @behaviour Stage + + @impl Stage + def runners, + do: [ + Runner.Transactions, + Runner.Transaction.Forks, + Runner.Logs, + Runner.Tokens, + Runner.TokenTransfers, + Runner.Address.TokenBalances + ] + + @impl Stage + def multis(runner_to_changes_list, options) do + {final_multi, final_remaining_runner_to_changes_list} = + Stage.single_multi(runners(), runner_to_changes_list, options) + + {[final_multi], final_remaining_runner_to_changes_list} + end +end diff --git a/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs b/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs index 5a5d547068..6167201f16 100644 --- a/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs +++ b/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs @@ -116,7 +116,7 @@ defmodule Explorer.Chain.Import.Runner.BlocksTest do assert count(Address.CurrentTokenBalance) == count end - test "remove_nonconsensus_data deletes token transfer rows with matching block number when new consensus block is inserted", + test "remove_nonconsensus_token_transfers deletes token transfer rows with matching block number when new consensus block is inserted", %{consensus_block: %{number: block_number} = block, options: options} do insert(:block, number: block_number, consensus: true) @@ -127,17 +127,15 @@ defmodule Explorer.Chain.Import.Runner.BlocksTest do assert {:ok, %{ - remove_nonconsensus_data: %{ - token_transfers: [ - %{transaction_hash: ^transaction_hash, log_index: ^log_index} - ] - } + remove_nonconsensus_token_transfers: [ + %{transaction_hash: ^transaction_hash, log_index: ^log_index} + ] }} = run_block_consensus_change(block, true, options) assert count(TokenTransfer) == 0 end - test "remove_nonconsensus_data does not delete token transfer rows with matching block number when new consensus block wasn't inserted", + test "remove_nonconsensus_token_transfers does not delete token transfer rows with matching block number when new consensus block wasn't inserted", %{consensus_block: %{number: block_number} = block, options: options} do insert(:token_transfer, block_number: block_number, transaction: insert(:transaction)) @@ -145,17 +143,12 @@ defmodule Explorer.Chain.Import.Runner.BlocksTest do assert count(TokenTransfer) == count - assert {:ok, - %{ - remove_nonconsensus_data: %{ - token_transfers: [] - } - }} = run_block_consensus_change(block, false, options) + assert {:ok, %{remove_nonconsensus_token_transfers: []}} = run_block_consensus_change(block, false, options) assert count(TokenTransfer) == count end - test "remove_nonconsensus_data deletes nonconsensus logs", %{ + test "remove_nonconsensus_logs deletes nonconsensus logs", %{ consensus_block: %{number: block_number} = block, options: options } do @@ -165,19 +158,13 @@ defmodule Explorer.Chain.Import.Runner.BlocksTest do assert count(Log) == 1 - assert {:ok, - %{ - remove_nonconsensus_data: %{ - logs: [ - %{transaction_hash: ^hash, index: ^index} - ] - } - }} = run_block_consensus_change(block, true, options) + assert {:ok, %{remove_nonconsensus_logs: [%{transaction_hash: ^hash, index: ^index}]}} = + run_block_consensus_change(block, true, options) assert count(Log) == 0 end - test "remove_nonconsensus_data deletes nonconsensus internal transactions", %{ + test "remove_nonconsensus_internal_transactions deletes nonconsensus internal transactions", %{ consensus_block: %{number: block_number} = block, options: options } do @@ -189,14 +176,8 @@ defmodule Explorer.Chain.Import.Runner.BlocksTest do assert count(InternalTransaction) == 1 - assert {:ok, - %{ - remove_nonconsensus_data: %{ - internal_transactions: [ - %{transaction_hash: ^hash, index: ^index} - ] - } - }} = run_block_consensus_change(block, true, options) + assert {:ok, %{remove_nonconsensus_internal_transactions: [%{transaction_hash: ^hash, index: ^index}]}} = + run_block_consensus_change(block, true, options) assert count(InternalTransaction) == 0 end diff --git a/docs/sharelocks.md b/docs/sharelocks.md index ebcc3570a6..389746a410 100644 --- a/docs/sharelocks.md +++ b/docs/sharelocks.md @@ -11,7 +11,13 @@ possible to incur in a deadlock and so into an error. This can be prevented by enforcing the same consistent order of lock aquisition on *all* the transactions performing `INSERT`, `UPDATE` or `DELETE` on a given table. -### Imposing the lock acquisition order with Ecto +On top of this, if multiple DB transactions act on multiple tables a deadlock +will occur, even if they follow the order on each table described above, if they +acquire locks on said tables in a different order. +This can also be prevented by using a consisten order of lock acquisition *between* +different tables. + +### Imposing the lock acquisition order on a table with Ecto When `INSERT`ing a list of rows Postgres will respect the order in which they appear in the query, so the reordering can happen beforehand. @@ -59,30 +65,73 @@ query = Repo.delete_all(from(e in Entry, join: s in subquery(query), on: e.id == s.id)) ``` -### Order used in Explorer's tables +### Imposing the lock acquisition order between tables with Ecto + +When using an `Ecto.Multi` to perform `INSERT`, `UPDATE` or `DELETE` on multiple +tables the order to keep is between different operation. +For example, supposing `EntryA` was established to be modified before `EntryB`, +this is not correct: +```elixir +Multi.new() +|> Multi.run(:update_b, fn repo, _ -> + # operations with ordered locks on `EntryB` +end) +|> Multi.run(:update_a, fn repo, _ -> + # operations with ordered locks on `EntryA` +end) +|> Repo.transaction() +``` + +When possible, the simple solution is to move `:update_a` to be before `:update_b`. +When not possible, for instance if `:update_a` depends on the result of `:update_b`, +this can be solved by acquiring the locks in a separate operation. + +For example: +```elixir +Multi.new() +|> Multi.run(:acquire_a, fn repo, _ -> + # acquire locks in order on `EntryA` +end) +|> Multi.run(:update_b, fn repo, _ -> + # operations with ordered locks on `EntryB` +end) +|> Multi.run(:update_a, fn repo, %{acquire_a: values} -> + # operations (no need to enforce order again) on `EntryA` +end) +|> Repo.transaction() +``` + +Note also that for the same reasons multiple operations on the same table in the +same transaction are not safe to perform if they each acquire locks in order, +because locks are not released until the transaction is committed. + +### Order used for Explorer's tables This is a complete list of the ordering currently in use on each table. +It also specifies the order between tables in the same transaction: locks for a +table on top need to be acquired before those from a table on the bottom. + Note that this should always be enforced because as long as there is one DB transaction performing in a different order there is the possibility of a deadlock. | schema module | table name | ordered by | |---------------|------------|------------| | Explorer.Chain.Address | addresses | asc: :hash | -| Explorer.Chain.Address.CoinBalance | address_coin_balances | [asc: :address_hash, asc: :block_number] | -| Explorer.Chain.Address.CurrentTokenBalance | address_current_token_balances | [asc: :address_hash, asc: :token_contract_address_hash] | | Explorer.Chain.Address.Name | address_names | [asc: :address_hash, asc: :name] | -| Explorer.Chain.Address.TokenBalance | address_token_balances | [asc: :address_hash, asc: :token_contract_address_hash, asc: :block_number] | +| Explorer.Chain.Address.CoinBalance | address_coin_balances | [asc: :address_hash, asc: :block_number] | | Explorer.Chain.Block | blocks | asc: :hash | -| Explorer.Chain.Block.EmissionReward | emission_rewards | asc: :block_range | -| Explorer.Chain.Block.Reward | block_rewards | [asc: :address_hash, asc: :address_type, asc: :block_hash] | | Explorer.Chain.Block.SecondDegreeRelation | block_second_degree_relations | [asc: :nephew_hash, asc: :uncle_hash] | -| Explorer.Chain.ContractMethod | contract_methods | [asc: :identified, asc: :abi] -| Explorer.Chain.InternalTransaction | internal_transactions | [asc: :transaction_hash, asc: :index] | +| Explorer.Chain.Block.Reward | block_rewards | [asc: :address_hash, asc: :address_type, asc: :block_hash] | +| Explorer.Chain.Block.EmissionReward | emission_rewards | asc: :block_range | +| Explorer.Chain.Transaction | transactions | asc: :hash | +| Explorer.Chain.Transaction.Fork | transaction_forks | [asc: :uncle_hash, asc: :index] | | Explorer.Chain.Log | logs | [asc: :transaction_hash, asc: :index] | -| Explorer.Chain.StakingPool | staking_pools | :staking_address_hash | -| Explorer.Chain.StakingPoolsDelegator | staking_pools_delegators | [asc: :delegator_address_hash, asc: :pool_address_hash] | +| Explorer.Chain.InternalTransaction | internal_transactions | [asc: :transaction_hash, asc: :index] | | Explorer.Chain.Token | tokens | asc: :contract_address_hash | | Explorer.Chain.TokenTransfer | token_transfers | [asc: :transaction_hash, asc: :log_index]| -| Explorer.Chain.Transaction | transactions | asc: :hash | -| Explorer.Chain.Transaction.Fork | transaction_forks | [asc: :uncle_hash, asc: :index] | +| Explorer.Chain.Address.TokenBalance | address_token_balances | [asc: :address_hash, asc: :token_contract_address_hash, asc: :block_number] | +| Explorer.Chain.Address.CurrentTokenBalance | address_current_token_balances | [asc: :address_hash, asc: :token_contract_address_hash] | +| Explorer.Chain.StakingPool | staking_pools | :staking_address_hash | +| Explorer.Chain.StakingPoolsDelegator | staking_pools_delegators | [asc: :delegator_address_hash, asc: :pool_address_hash] | +| Explorer.Chain.ContractMethod | contract_methods | [asc: :identified, asc: :abi] | Explorer.Market.MarketHistory | market_history | asc: :date | From 0bfa84d7819f52ebcf1a3738e97f66b01f3dffd4 Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Wed, 18 Sep 2019 14:32:48 +0300 Subject: [PATCH 45/67] Update CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a403e6b61c..51b2541a22 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ - [#2684](https://github.com/poanetwork/blockscout/pull/2684) - do not filter pending logs - [#2682](https://github.com/poanetwork/blockscout/pull/2682) - Use Task.start instead of Task.async in caches - [#2671](https://github.com/poanetwork/blockscout/pull/2671) - fixed buttons color at smart contract section +- [#2619](https://github.com/poanetwork/blockscout/pull/2619) - Enforce DB transaction's order to prevent deadlocks ### Chore @@ -31,7 +32,6 @@ - [#2497](https://github.com/poanetwork/blockscout/pull/2497) - Add generic Ordered Cache behaviour and implementation ### Fixes -- [#2619](https://github.com/poanetwork/blockscout/pull/2619) - Enforce DB transaction's order to prevent deadlocks - [#2659](https://github.com/poanetwork/blockscout/pull/2659) - Multipurpose front-end part update - [#2640](https://github.com/poanetwork/blockscout/pull/2640) - SVG network icons - [#2635](https://github.com/poanetwork/blockscout/pull/2635) - optimize ERC721 inventory query From 4c8953a883576488e3ba0c104ac1c30e23919bb3 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Wed, 18 Sep 2019 15:13:36 +0300 Subject: [PATCH 46/67] fix empty total_supply in coin gecko response --- apps/explorer/lib/explorer/exchange_rates/source/coin_gecko.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/explorer/lib/explorer/exchange_rates/source/coin_gecko.ex b/apps/explorer/lib/explorer/exchange_rates/source/coin_gecko.ex index 9d4cfba3f4..b25cc53ff5 100644 --- a/apps/explorer/lib/explorer/exchange_rates/source/coin_gecko.ex +++ b/apps/explorer/lib/explorer/exchange_rates/source/coin_gecko.ex @@ -26,7 +26,7 @@ defmodule Explorer.ExchangeRates.Source.CoinGecko do [ %Token{ available_supply: to_decimal(market_data["circulating_supply"]), - total_supply: to_decimal(market_data["total_supply"]), + total_supply: to_decimal(market_data["total_supply"]) || to_decimal(market_data["circulating_supply"]), btc_value: btc_value, id: json_data["id"], last_updated: last_updated, From 66503e4164b0460b1b731171dd6c040d4b1ddbcd Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Wed, 18 Sep 2019 15:15:31 +0300 Subject: [PATCH 47/67] add CHANGELOG entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f3a0f91771..c818adeda8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - [#2663](https://github.com/poanetwork/blockscout/pull/2663) - Fetch address counters in parallel ### Fixes +- [#2706](https://github.com/poanetwork/blockscout/pull/2706) - fix empty total_supply in coin gecko response - [#2701](https://github.com/poanetwork/blockscout/pull/2701) - Exclude nonconsensus blocks from avg block time calculation by default - [#2696](https://github.com/poanetwork/blockscout/pull/2696) - do not update fetched_coin_balance with nil - [#2693](https://github.com/poanetwork/blockscout/pull/2693) - remove non consensus internal transactions From 5e0b5308b86f59daab0c8595b8dccc6076c279d3 Mon Sep 17 00:00:00 2001 From: Yegor San Date: Wed, 18 Sep 2019 15:27:47 +0300 Subject: [PATCH 48/67] canvas --- .../assets/css/components/_dashboard-banner.scss | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/block_scout_web/assets/css/components/_dashboard-banner.scss b/apps/block_scout_web/assets/css/components/_dashboard-banner.scss index 0cf8071caf..bae39d46a1 100644 --- a/apps/block_scout_web/assets/css/components/_dashboard-banner.scss +++ b/apps/block_scout_web/assets/css/components/_dashboard-banner.scss @@ -72,7 +72,7 @@ $dashboard-banner-chart-axis-font-color: $dashboard-stats-item-value-color !defa } .dashboard-banner-chart-legend { - display: flex; + display: grid; grid-template-columns: 1fr 1fr; padding-bottom: 12px; @@ -81,7 +81,7 @@ $dashboard-banner-chart-axis-font-color: $dashboard-stats-item-value-color !defa padding-left: 12px; padding-top: 3px; position: relative; - padding-right: 60px; + padding-right: 12px; @include media-breakpoint-down(md) { display: flex; From 55339cb1c10e05ece1e3c16732490855287ad957 Mon Sep 17 00:00:00 2001 From: Yegor San Date: Wed, 18 Sep 2019 15:49:03 +0300 Subject: [PATCH 49/67] Updated Changelog.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f794cf3f8..04fb795636 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - [#2663](https://github.com/poanetwork/blockscout/pull/2663) - Fetch address counters in parallel ### Fixes +- [#2707](https://github.com/poanetwork/blockscout/pull/2707) - fix for dashboard banner chart legend items - [#2701](https://github.com/poanetwork/blockscout/pull/2701) - Exclude nonconsensus blocks from avg block time calculation by default - [#2696](https://github.com/poanetwork/blockscout/pull/2696) - do not update fetched_coin_balance with nil - [#2693](https://github.com/poanetwork/blockscout/pull/2693) - remove non consensus internal transactions From af7782f2fe46983159f9e8e00966e57cb1721e8f Mon Sep 17 00:00:00 2001 From: pasqu4le Date: Wed, 4 Sep 2019 18:29:57 +0200 Subject: [PATCH 50/67] Add ETS-based cache for accounts page --- CHANGELOG.md | 1 + .../block_scout_web/test/support/conn_case.ex | 2 + .../test/support/feature_case.ex | 2 + apps/explorer/config/config.exs | 4 + apps/explorer/lib/explorer/application.ex | 4 +- apps/explorer/lib/explorer/chain.ex | 31 ++++++++ .../lib/explorer/chain/cache/accounts.ex | 73 +++++++++++++++++++ .../explorer/chain/cache/accounts_test.exs | 42 +++++++++++ apps/explorer/test/support/data_case.ex | 2 + apps/indexer/lib/indexer/block/fetcher.ex | 5 +- .../lib/indexer/block/realtime/fetcher.ex | 3 + .../lib/indexer/fetcher/block_reward.ex | 5 +- .../lib/indexer/fetcher/coin_balance.ex | 5 +- .../indexer/fetcher/coin_balance_on_demand.ex | 8 +- .../lib/indexer/fetcher/contract_code.ex | 4 +- .../indexer/fetcher/internal_transaction.ex | 3 + .../indexer/fetcher/pending_transaction.ex | 4 +- .../lib/indexer/fetcher/uncle_block.ex | 4 +- 18 files changed, 193 insertions(+), 9 deletions(-) create mode 100644 apps/explorer/lib/explorer/chain/cache/accounts.ex create mode 100644 apps/explorer/test/explorer/chain/cache/accounts_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index f3a0f91771..53299264df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## Current ### Features +- [#2667](https://github.com/poanetwork/blockscout/pull/2667) - Add ETS-based cache for accounts page - [#2679](https://github.com/poanetwork/blockscout/pull/2679) - added fixed height for card chain blocks and card chain transactions - [#2678](https://github.com/poanetwork/blockscout/pull/2678) - fixed dashboard banner height bug - [#2672](https://github.com/poanetwork/blockscout/pull/2672) - added new theme for xUSDT diff --git a/apps/block_scout_web/test/support/conn_case.ex b/apps/block_scout_web/test/support/conn_case.ex index 4ab8f3bcd9..ed3f094c34 100644 --- a/apps/block_scout_web/test/support/conn_case.ex +++ b/apps/block_scout_web/test/support/conn_case.ex @@ -42,6 +42,8 @@ defmodule BlockScoutWeb.ConnCase do Supervisor.terminate_child(Explorer.Supervisor, Explorer.Chain.Cache.Transactions.child_id()) Supervisor.restart_child(Explorer.Supervisor, Explorer.Chain.Cache.Transactions.child_id()) + Supervisor.terminate_child(Explorer.Supervisor, Explorer.Chain.Cache.Accounts.child_id()) + Supervisor.restart_child(Explorer.Supervisor, Explorer.Chain.Cache.Accounts.child_id()) {:ok, conn: Phoenix.ConnTest.build_conn()} end diff --git a/apps/block_scout_web/test/support/feature_case.ex b/apps/block_scout_web/test/support/feature_case.ex index cdacd547bf..f6b1aedf24 100644 --- a/apps/block_scout_web/test/support/feature_case.ex +++ b/apps/block_scout_web/test/support/feature_case.ex @@ -29,6 +29,8 @@ defmodule BlockScoutWeb.FeatureCase do Supervisor.terminate_child(Explorer.Supervisor, Explorer.Chain.Cache.Transactions.child_id()) Supervisor.restart_child(Explorer.Supervisor, Explorer.Chain.Cache.Transactions.child_id()) + Supervisor.terminate_child(Explorer.Supervisor, Explorer.Chain.Cache.Accounts.child_id()) + Supervisor.restart_child(Explorer.Supervisor, Explorer.Chain.Cache.Accounts.child_id()) metadata = Phoenix.Ecto.SQL.Sandbox.metadata_for(Explorer.Repo, self()) {:ok, session} = Wallaby.start_session(metadata: metadata) diff --git a/apps/explorer/config/config.exs b/apps/explorer/config/config.exs index 8a264eaf33..48d9b55610 100644 --- a/apps/explorer/config/config.exs +++ b/apps/explorer/config/config.exs @@ -136,6 +136,10 @@ config :explorer, Explorer.Chain.Cache.Transactions, ttl_check_interval: if(System.get_env("DISABLE_INDEXER") == "true", do: :timer.seconds(1), else: false), global_ttl: if(System.get_env("DISABLE_INDEXER") == "true", do: :timer.seconds(5)) +config :explorer, Explorer.Chain.Cache.Accounts, + ttl_check_interval: if(System.get_env("DISABLE_INDEXER") == "true", do: :timer.seconds(1), else: false), + global_ttl: if(System.get_env("DISABLE_INDEXER") == "true", do: :timer.seconds(5)) + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" diff --git a/apps/explorer/lib/explorer/application.ex b/apps/explorer/lib/explorer/application.ex index 7c55d28f76..9645572b62 100644 --- a/apps/explorer/lib/explorer/application.ex +++ b/apps/explorer/lib/explorer/application.ex @@ -8,6 +8,7 @@ defmodule Explorer.Application do alias Explorer.Admin alias Explorer.Chain.Cache.{ + Accounts, BlockCount, BlockNumber, Blocks, @@ -49,7 +50,8 @@ defmodule Explorer.Application do BlockNumber, con_cache_child_spec(MarketHistoryCache.cache_name()), con_cache_child_spec(RSK.cache_name(), ttl_check_interval: :timer.minutes(1), global_ttl: :timer.minutes(30)), - Transactions + Transactions, + Accounts ] children = base_children ++ configurable_children() diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index 6de1adc9ff..d39945034d 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -49,6 +49,7 @@ defmodule Explorer.Chain do alias Explorer.Chain.Block.{EmissionReward, Reward} alias Explorer.Chain.Cache.{ + Accounts, BlockCount, BlockNumber, Blocks, @@ -1379,6 +1380,36 @@ defmodule Explorer.Chain do def list_top_addresses(options \\ []) do paging_options = Keyword.get(options, :paging_options, @default_paging_options) + if is_nil(paging_options.key) do + paging_options.page_size + |> Accounts.take_enough() + |> case do + nil -> + accounts_with_n = fetch_top_addresses(paging_options) + + accounts_with_n + |> Enum.map(fn {address, _n} -> address end) + |> Accounts.update() + + accounts_with_n + + accounts -> + Enum.map( + accounts, + &{&1, + if is_nil(&1.nonce) do + 0 + else + &1.nonce + 1 + end} + ) + end + else + fetch_top_addresses(paging_options) + end + end + + defp fetch_top_addresses(paging_options) do base_query = from(a in Address, where: a.fetched_coin_balance > ^0, diff --git a/apps/explorer/lib/explorer/chain/cache/accounts.ex b/apps/explorer/lib/explorer/chain/cache/accounts.ex new file mode 100644 index 0000000000..523bd44877 --- /dev/null +++ b/apps/explorer/lib/explorer/chain/cache/accounts.ex @@ -0,0 +1,73 @@ +defmodule Explorer.Chain.Cache.Accounts do + @moduledoc """ + Caches the top Addresses + """ + + alias Explorer.Chain.Address + + use Explorer.Chain.OrderedCache, + name: :accounts, + max_size: 51, + preload: :names, + ttl_check_interval: Application.get_env(:explorer, __MODULE__)[:ttl_check_interval], + global_ttl: Application.get_env(:explorer, __MODULE__)[:global_ttl] + + @type element :: Address.t() + + @type id :: {non_neg_integer(), non_neg_integer()} + + def element_to_id(%Address{fetched_coin_balance: fetched_coin_balance, hash: hash}) do + {fetched_coin_balance, hash} + end + + def prevails?({fetched_coin_balance_a, hash_a}, {fetched_coin_balance_b, hash_b}) do + # same as a query's `order_by: [desc: :fetched_coin_balance, asc: :hash]` + if fetched_coin_balance_a == fetched_coin_balance_b do + hash_a < hash_b + else + fetched_coin_balance_a > fetched_coin_balance_b + end + end + + def drop(nil), do: :ok + + def drop([]), do: :ok + + def drop(addresses) when is_list(addresses) do + # This has to be used by the Indexer insead of `update`. + # The reason being that addresses already in the cache can change their balance + # value and removing or updating them will result into a potentially invalid + # cache status, that would not even get corrected with time. + # The only thing we can safely do when an address in the cache changes its + # `fetched_coin_balance` is to invalidate the whole cache and wait for it + # to be filled again (by the query that it takes the place of when full). + ConCache.update(cache_name(), ids_list_key(), fn ids -> + if drop_needed?(ids, addresses) do + # Remove the addresses immediately + Enum.each(ids, &ConCache.delete(cache_name(), &1)) + + {:ok, []} + else + {:ok, ids} + end + end) + end + + def drop(address), do: drop([address]) + + defp drop_needed?(ids, _addresses) when is_nil(ids), do: false + + defp drop_needed?([], _addresses), do: false + + defp drop_needed?(ids, addresses) do + ids_map = Map.new(ids, fn {balance, hash} -> {hash, balance} end) + + # Result it `true` only when the address is present in the cache already, + # but with a different `fetched_coin_balance` + Enum.find_value(addresses, false, fn address -> + stored_address_balance = Map.get(ids_map, address.hash) + + stored_address_balance && stored_address_balance != address.fetched_coin_balance + end) + end +end diff --git a/apps/explorer/test/explorer/chain/cache/accounts_test.exs b/apps/explorer/test/explorer/chain/cache/accounts_test.exs new file mode 100644 index 0000000000..a1a49bd67b --- /dev/null +++ b/apps/explorer/test/explorer/chain/cache/accounts_test.exs @@ -0,0 +1,42 @@ +defmodule Explorer.Chain.Cache.AccountsTest do + use Explorer.DataCase + + alias Explorer.Chain.Cache.Accounts + alias Explorer.Repo + + describe "drop/1" do + test "does not drop the cache if the address fetched_coin_balance has not changed" do + address = + insert(:address, fetched_coin_balance: 100_000, fetched_coin_balance_block_number: 1) + |> preload_names() + + Accounts.update(address) + + assert Accounts.take(1) == [address] + + Accounts.drop(address) + + assert Accounts.take(1) == [address] + end + + test "drops the cache if an address was in the cache with a different fetched_coin_balance" do + address = + insert(:address, fetched_coin_balance: 100_000, fetched_coin_balance_block_number: 1) + |> preload_names() + + Accounts.update(address) + + assert Accounts.take(1) == [address] + + updated_address = %{address | fetched_coin_balance: 100_001} + + Accounts.drop(updated_address) + + assert Accounts.take(1) == [] + end + end + + defp preload_names(address) do + Repo.preload(address, [:names]) + end +end diff --git a/apps/explorer/test/support/data_case.ex b/apps/explorer/test/support/data_case.ex index 47c15bbca0..e12dd24a82 100644 --- a/apps/explorer/test/support/data_case.ex +++ b/apps/explorer/test/support/data_case.ex @@ -45,6 +45,8 @@ defmodule Explorer.DataCase do Supervisor.restart_child(Explorer.Supervisor, Explorer.Chain.Cache.Blocks.child_id()) Supervisor.terminate_child(Explorer.Supervisor, Explorer.Chain.Cache.Transactions.child_id()) Supervisor.restart_child(Explorer.Supervisor, Explorer.Chain.Cache.Transactions.child_id()) + Supervisor.terminate_child(Explorer.Supervisor, Explorer.Chain.Cache.Accounts.child_id()) + Supervisor.restart_child(Explorer.Supervisor, Explorer.Chain.Cache.Accounts.child_id()) :ok end diff --git a/apps/indexer/lib/indexer/block/fetcher.ex b/apps/indexer/lib/indexer/block/fetcher.ex index 28002f010a..1b95d84086 100644 --- a/apps/indexer/lib/indexer/block/fetcher.ex +++ b/apps/indexer/lib/indexer/block/fetcher.ex @@ -13,7 +13,7 @@ defmodule Indexer.Block.Fetcher do alias Explorer.Chain alias Explorer.Chain.{Address, Block, Hash, Import, Transaction} alias Explorer.Chain.Cache.Blocks, as: BlocksCache - alias Explorer.Chain.Cache.{BlockNumber, Transactions} + alias Explorer.Chain.Cache.{Accounts, BlockNumber, Transactions} alias Indexer.Block.Fetcher.Receipts alias Indexer.Fetcher.{ @@ -176,6 +176,7 @@ defmodule Indexer.Block.Fetcher do result = {:ok, %{inserted: inserted, errors: blocks_errors}} update_block_cache(inserted[:blocks]) update_transactions_cache(inserted[:transactions]) + update_addresses_cache(inserted[:addresses]) result else {step, {:error, reason}} -> {:error, {step, reason}} @@ -197,6 +198,8 @@ defmodule Indexer.Block.Fetcher do Transactions.update(transactions) end + defp update_addresses_cache(addresses), do: Accounts.drop(addresses) + def import( %__MODULE__{broadcast: broadcast, callback_module: callback_module} = state, options diff --git a/apps/indexer/lib/indexer/block/realtime/fetcher.ex b/apps/indexer/lib/indexer/block/realtime/fetcher.ex index 8edd87738d..897eab2cff 100644 --- a/apps/indexer/lib/indexer/block/realtime/fetcher.ex +++ b/apps/indexer/lib/indexer/block/realtime/fetcher.ex @@ -27,6 +27,7 @@ defmodule Indexer.Block.Realtime.Fetcher do alias Ecto.Changeset alias EthereumJSONRPC.{FetchedBalances, Subscription} alias Explorer.Chain + alias Explorer.Chain.Cache.Accounts alias Explorer.Counters.AverageBlockTime alias Indexer.{Block, Tracer} alias Indexer.Block.Realtime.TaskSupervisor @@ -197,6 +198,8 @@ defmodule Indexer.Block.Realtime.Fetcher do json_rpc_named_arguments ) + Accounts.drop(imported[:addresses]) + ok end end diff --git a/apps/indexer/lib/indexer/fetcher/block_reward.ex b/apps/indexer/lib/indexer/fetcher/block_reward.ex index 4a59eb1cbe..ed593bb655 100644 --- a/apps/indexer/lib/indexer/fetcher/block_reward.ex +++ b/apps/indexer/lib/indexer/fetcher/block_reward.ex @@ -17,6 +17,7 @@ defmodule Indexer.Fetcher.BlockReward do alias EthereumJSONRPC.FetchedBeneficiaries alias Explorer.Chain alias Explorer.Chain.{Block, Wei} + alias Explorer.Chain.Cache.Accounts alias Indexer.{BufferedTask, Tracer} alias Indexer.Fetcher.BlockReward.Supervisor, as: BlockRewardSupervisor alias Indexer.Fetcher.CoinBalance @@ -130,7 +131,9 @@ defmodule Indexer.Fetcher.BlockReward do |> add_gas_payments() |> import_block_reward_params() |> case do - {:ok, %{address_coin_balances: address_coin_balances}} -> + {:ok, %{address_coin_balances: address_coin_balances, addresses: addresses}} -> + Accounts.drop(addresses) + CoinBalance.async_fetch_balances(address_coin_balances) retry_errors(errors) diff --git a/apps/indexer/lib/indexer/fetcher/coin_balance.ex b/apps/indexer/lib/indexer/fetcher/coin_balance.ex index e9930d0e0e..f8fadfcb53 100644 --- a/apps/indexer/lib/indexer/fetcher/coin_balance.ex +++ b/apps/indexer/lib/indexer/fetcher/coin_balance.ex @@ -14,6 +14,7 @@ defmodule Indexer.Fetcher.CoinBalance do alias EthereumJSONRPC.FetchedBalances alias Explorer.Chain alias Explorer.Chain.{Block, Hash} + alias Explorer.Chain.Cache.Accounts alias Indexer.{BufferedTask, Tracer} @behaviour BufferedTask @@ -136,7 +137,9 @@ defmodule Indexer.Fetcher.CoinBalance do end defp run_fetched_balances(%FetchedBalances{errors: errors} = fetched_balances, _) do - {:ok, _} = import_fetched_balances(fetched_balances) + {:ok, imported} = import_fetched_balances(fetched_balances) + + Accounts.drop(imported[:addresses]) retry(errors) end diff --git a/apps/indexer/lib/indexer/fetcher/coin_balance_on_demand.ex b/apps/indexer/lib/indexer/fetcher/coin_balance_on_demand.ex index d6d7d66d71..17014d2ede 100644 --- a/apps/indexer/lib/indexer/fetcher/coin_balance_on_demand.ex +++ b/apps/indexer/lib/indexer/fetcher/coin_balance_on_demand.ex @@ -19,7 +19,7 @@ defmodule Indexer.Fetcher.CoinBalanceOnDemand do alias Explorer.{Chain, Repo} alias Explorer.Chain.Address alias Explorer.Chain.Address.CoinBalance - alias Explorer.Chain.Cache.BlockNumber + alias Explorer.Chain.Cache.{Accounts, BlockNumber} alias Explorer.Counters.AverageBlockTime alias Indexer.Fetcher.CoinBalance, as: CoinBalanceFetcher alias Timex.Duration @@ -71,7 +71,11 @@ defmodule Indexer.Fetcher.CoinBalanceOnDemand do end def handle_cast({:fetch_and_update, block_number, address}, state) do - fetch_and_update(block_number, address, state.json_rpc_named_arguments) + result = fetch_and_update(block_number, address, state.json_rpc_named_arguments) + + with {:ok, %{addresses: addresses}} <- result do + Accounts.drop(addresses) + end {:noreply, state} end diff --git a/apps/indexer/lib/indexer/fetcher/contract_code.ex b/apps/indexer/lib/indexer/fetcher/contract_code.ex index bc95dd0d8e..cd9a5baa08 100644 --- a/apps/indexer/lib/indexer/fetcher/contract_code.ex +++ b/apps/indexer/lib/indexer/fetcher/contract_code.ex @@ -12,6 +12,7 @@ defmodule Indexer.Fetcher.ContractCode do alias Explorer.Chain alias Explorer.Chain.{Block, Hash} + alias Explorer.Chain.Cache.Accounts alias Indexer.{BufferedTask, Tracer} alias Indexer.Transform.Addresses @@ -126,7 +127,8 @@ defmodule Indexer.Fetcher.ContractCode do addresses: %{params: merged_addresses_params}, timeout: :infinity }) do - {:ok, _} -> + {:ok, imported} -> + Accounts.drop(imported[:addresses]) :ok {:error, step, reason, _changes_so_far} -> diff --git a/apps/indexer/lib/indexer/fetcher/internal_transaction.ex b/apps/indexer/lib/indexer/fetcher/internal_transaction.ex index 7d93bbf64f..5ad7c5d506 100644 --- a/apps/indexer/lib/indexer/fetcher/internal_transaction.ex +++ b/apps/indexer/lib/indexer/fetcher/internal_transaction.ex @@ -14,6 +14,7 @@ defmodule Indexer.Fetcher.InternalTransaction do alias Explorer.Chain alias Explorer.Chain.{Block, Hash} + alias Explorer.Chain.Cache.Accounts alias Indexer.{BufferedTask, Tracer} alias Indexer.Transform.Addresses @@ -218,6 +219,8 @@ defmodule Indexer.Fetcher.InternalTransaction do case imports do {:ok, imported} -> + Accounts.drop(imported[:addreses]) + async_import_coin_balances(imported, %{ address_hash_to_fetched_balance_block_number: address_hash_to_block_number }) diff --git a/apps/indexer/lib/indexer/fetcher/pending_transaction.ex b/apps/indexer/lib/indexer/fetcher/pending_transaction.ex index 7576d384d8..176d8dc9b7 100644 --- a/apps/indexer/lib/indexer/fetcher/pending_transaction.ex +++ b/apps/indexer/lib/indexer/fetcher/pending_transaction.ex @@ -14,6 +14,7 @@ defmodule Indexer.Fetcher.PendingTransaction do alias Ecto.Changeset alias Explorer.Chain + alias Explorer.Chain.Cache.Accounts alias Indexer.Fetcher.PendingTransaction alias Indexer.Transform.Addresses @@ -148,7 +149,8 @@ defmodule Indexer.Fetcher.PendingTransaction do broadcast: :realtime, transactions: %{params: transactions_params, on_conflict: :nothing} }) do - {:ok, _} -> + {:ok, imported} -> + Accounts.drop(imported[:addresses]) :ok {:error, [%Changeset{} | _] = changesets} -> diff --git a/apps/indexer/lib/indexer/fetcher/uncle_block.ex b/apps/indexer/lib/indexer/fetcher/uncle_block.ex index e9300d1e39..e597d16ca2 100644 --- a/apps/indexer/lib/indexer/fetcher/uncle_block.ex +++ b/apps/indexer/lib/indexer/fetcher/uncle_block.ex @@ -12,6 +12,7 @@ defmodule Indexer.Fetcher.UncleBlock do alias Ecto.Changeset alias EthereumJSONRPC.Blocks alias Explorer.Chain + alias Explorer.Chain.Cache.Accounts alias Explorer.Chain.Hash alias Indexer.{Block, BufferedTask, Tracer} alias Indexer.Fetcher.UncleBlock @@ -126,7 +127,8 @@ defmodule Indexer.Fetcher.UncleBlock do block_second_degree_relations: %{params: block_second_degree_relations_params}, transactions: %{params: transactions_params, on_conflict: :nothing} }) do - {:ok, _} -> + {:ok, imported} -> + Accounts.drop(imported[:addresses]) retry(errors) {:error, {:import = step, [%Changeset{} | _] = changesets}} -> From bde1c51268276867c8a01e118e8e00db19ef933a Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Wed, 18 Sep 2019 16:40:37 +0300 Subject: [PATCH 51/67] add log index to logs view --- .../templates/transaction_log/_logs.html.eex | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/apps/block_scout_web/lib/block_scout_web/templates/transaction_log/_logs.html.eex b/apps/block_scout_web/lib/block_scout_web/templates/transaction_log/_logs.html.eex index 49a18bf84d..6615ec8259 100644 --- a/apps/block_scout_web/lib/block_scout_web/templates/transaction_log/_logs.html.eex +++ b/apps/block_scout_web/lib/block_scout_web/templates/transaction_log/_logs.html.eex @@ -185,5 +185,13 @@
<% end %> +
+ <%= gettext "Log Index" %> +
+
+
+ <%= @log.index %> +
+
From ce59ec391a2686f4ed723cc3b2bcafd122575900 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Wed, 18 Sep 2019 16:48:11 +0300 Subject: [PATCH 52/67] add CHANGELOG entry --- CHANGELOG.md | 1 + apps/block_scout_web/priv/gettext/default.pot | 5 +++++ apps/block_scout_web/priv/gettext/en/LC_MESSAGES/default.po | 5 +++++ 3 files changed, 11 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bcd5efbc8f..bc5ddb9e1f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ - [#2619](https://github.com/poanetwork/blockscout/pull/2619) - Enforce DB transaction's order to prevent deadlocks ### Chore +- [#2708](https://github.com/poanetwork/blockscout/pull/2708) - add log index to logs view ## 2.0.4-beta diff --git a/apps/block_scout_web/priv/gettext/default.pot b/apps/block_scout_web/priv/gettext/default.pot index 8a1582b2e7..f7628dcff4 100644 --- a/apps/block_scout_web/priv/gettext/default.pot +++ b/apps/block_scout_web/priv/gettext/default.pot @@ -1841,3 +1841,8 @@ msgstr "" #: lib/block_scout_web/views/address_contract_view.ex:22 msgid "true" msgstr "" + +#, elixir-format +#: lib/block_scout_web/templates/transaction_log/_logs.html.eex:189 +msgid "Log Index" +msgstr "" diff --git a/apps/block_scout_web/priv/gettext/en/LC_MESSAGES/default.po b/apps/block_scout_web/priv/gettext/en/LC_MESSAGES/default.po index 8a1582b2e7..f7628dcff4 100644 --- a/apps/block_scout_web/priv/gettext/en/LC_MESSAGES/default.po +++ b/apps/block_scout_web/priv/gettext/en/LC_MESSAGES/default.po @@ -1841,3 +1841,8 @@ msgstr "" #: lib/block_scout_web/views/address_contract_view.ex:22 msgid "true" msgstr "" + +#, elixir-format +#: lib/block_scout_web/templates/transaction_log/_logs.html.eex:189 +msgid "Log Index" +msgstr "" From 432e6419dfcac6d7a1a400608cfa8f671824f708 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Wed, 18 Sep 2019 17:36:27 +0300 Subject: [PATCH 53/67] add comment --- apps/explorer/lib/explorer/chain.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index 1951f500a2..ce6995fc01 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -3074,6 +3074,7 @@ defmodule Explorer.Chain do |> normalize_balances_by_day() end + # https://github.com/poanetwork/blockscout/issues/2658 defp replace_last_value(items, %{value: value, timestamp: timestamp}) do List.replace_at(items, -1, %{date: Date.convert!(timestamp, Calendar.ISO), value: value}) end From 24d606dbed40203de7b4ef58cea6afb5e164d939 Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Wed, 18 Sep 2019 17:40:34 +0300 Subject: [PATCH 54/67] Fix stuck label and value for uncle block height --- CHANGELOG.md | 1 + .../templates/block/overview.html.eex | 3 +- apps/block_scout_web/priv/gettext/default.pot | 28 +++++++++---------- .../priv/gettext/en/LC_MESSAGES/default.po | 28 +++++++++---------- 4 files changed, 30 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 30f508dbd2..d98f2c0141 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ - [#2663](https://github.com/poanetwork/blockscout/pull/2663) - Fetch address counters in parallel ### Fixes +- [#2709](https://github.com/poanetwork/blockscout/pull/2709) - Fix stuck label and value for uncle block height - [#2707](https://github.com/poanetwork/blockscout/pull/2707) - fix for dashboard banner chart legend items - [#2701](https://github.com/poanetwork/blockscout/pull/2701) - Exclude nonconsensus blocks from avg block time calculation by default - [#2696](https://github.com/poanetwork/blockscout/pull/2696) - do not update fetched_coin_balance with nil diff --git a/apps/block_scout_web/lib/block_scout_web/templates/block/overview.html.eex b/apps/block_scout_web/lib/block_scout_web/templates/block/overview.html.eex index 752fd314e3..c2774ce4bd 100644 --- a/apps/block_scout_web/lib/block_scout_web/templates/block/overview.html.eex +++ b/apps/block_scout_web/lib/block_scout_web/templates/block/overview.html.eex @@ -12,8 +12,7 @@ <%= if block_type(@block) == "Block" do %> <%= gettext("Block Height: %{height}", height: @block.number) %> <%= if @block.number == 0, do: "- " <> gettext("Genesis Block")%> <% else %> - <%= gettext("%{block_type} Height:", block_type: block_type(@block)) %> - <%= link(@block, to: block_path(BlockScoutWeb.Endpoint, :show, @block.number)) %> + <%= gettext("%{block_type} Height:", block_type: block_type(@block)) %> <%= link(@block, to: block_path(BlockScoutWeb.Endpoint, :show, @block.number)) %> <% end %>
diff --git a/apps/block_scout_web/priv/gettext/default.pot b/apps/block_scout_web/priv/gettext/default.pot index 8a1582b2e7..dfe785c39b 100644 --- a/apps/block_scout_web/priv/gettext/default.pot +++ b/apps/block_scout_web/priv/gettext/default.pot @@ -43,7 +43,7 @@ msgid "%{block_type}s" msgstr "" #, elixir-format -#: lib/block_scout_web/templates/block/overview.html.eex:21 +#: lib/block_scout_web/templates/block/overview.html.eex:20 #: lib/block_scout_web/templates/chain/_block.html.eex:11 msgid "%{count} Transactions" msgstr "" @@ -480,7 +480,7 @@ msgid "Copy Txn Hash" msgstr "" #, elixir-format -#: lib/block_scout_web/templates/block/overview.html.eex:59 +#: lib/block_scout_web/templates/block/overview.html.eex:58 msgid "Difficulty" msgstr "" @@ -562,15 +562,15 @@ msgstr "" #, elixir-format #: lib/block_scout_web/templates/block/_tile.html.eex:56 -#: lib/block_scout_web/templates/block/overview.html.eex:109 -#: lib/block_scout_web/templates/block/overview.html.eex:159 +#: lib/block_scout_web/templates/block/overview.html.eex:108 +#: lib/block_scout_web/templates/block/overview.html.eex:158 msgid "Gas Limit" msgstr "" #, elixir-format #: lib/block_scout_web/templates/block/_tile.html.eex:61 -#: lib/block_scout_web/templates/block/overview.html.eex:102 -#: lib/block_scout_web/templates/block/overview.html.eex:153 +#: lib/block_scout_web/templates/block/overview.html.eex:101 +#: lib/block_scout_web/templates/block/overview.html.eex:152 msgid "Gas Used" msgstr "" @@ -657,7 +657,7 @@ msgstr "" #, elixir-format #: lib/block_scout_web/templates/block/_tile.html.eex:38 -#: lib/block_scout_web/templates/block/overview.html.eex:125 +#: lib/block_scout_web/templates/block/overview.html.eex:124 #: lib/block_scout_web/templates/chain/_block.html.eex:15 msgid "Miner" msgstr "" @@ -712,7 +712,7 @@ msgid "Execute" msgstr "" #, elixir-format -#: lib/block_scout_web/templates/block/overview.html.eex:74 +#: lib/block_scout_web/templates/block/overview.html.eex:73 #: lib/block_scout_web/templates/transaction/overview.html.eex:79 msgid "Nonce" msgstr "" @@ -767,7 +767,7 @@ msgid "GET" msgstr "" #, elixir-format -#: lib/block_scout_web/templates/block/overview.html.eex:85 +#: lib/block_scout_web/templates/block/overview.html.eex:84 msgid "Position %{index}" msgstr "" @@ -798,7 +798,7 @@ msgid "Gwei" msgstr "" #, elixir-format -#: lib/block_scout_web/templates/block/overview.html.eex:37 +#: lib/block_scout_web/templates/block/overview.html.eex:36 msgid "Hash" msgstr "" @@ -964,7 +964,7 @@ msgid "Top Accounts - %{subnetwork} Explorer" msgstr "" #, elixir-format -#: lib/block_scout_web/templates/block/overview.html.eex:68 +#: lib/block_scout_web/templates/block/overview.html.eex:67 msgid "Total Difficulty" msgstr "" @@ -1246,7 +1246,7 @@ msgid "Parameters" msgstr "" #, elixir-format -#: lib/block_scout_web/templates/block/overview.html.eex:44 +#: lib/block_scout_web/templates/block/overview.html.eex:43 msgid "Parent Hash" msgstr "" @@ -1404,7 +1404,7 @@ msgid "Show Validator Info" msgstr "" #, elixir-format -#: lib/block_scout_web/templates/block/overview.html.eex:146 +#: lib/block_scout_web/templates/block/overview.html.eex:145 msgid "Block Rewards" msgstr "" @@ -1646,7 +1646,7 @@ msgid "Uncle Reward" msgstr "" #, elixir-format -#: lib/block_scout_web/templates/block/overview.html.eex:81 +#: lib/block_scout_web/templates/block/overview.html.eex:80 #: lib/block_scout_web/templates/layout/_topnav.html.eex:52 msgid "Uncles" msgstr "" diff --git a/apps/block_scout_web/priv/gettext/en/LC_MESSAGES/default.po b/apps/block_scout_web/priv/gettext/en/LC_MESSAGES/default.po index 8a1582b2e7..dfe785c39b 100644 --- a/apps/block_scout_web/priv/gettext/en/LC_MESSAGES/default.po +++ b/apps/block_scout_web/priv/gettext/en/LC_MESSAGES/default.po @@ -43,7 +43,7 @@ msgid "%{block_type}s" msgstr "" #, elixir-format -#: lib/block_scout_web/templates/block/overview.html.eex:21 +#: lib/block_scout_web/templates/block/overview.html.eex:20 #: lib/block_scout_web/templates/chain/_block.html.eex:11 msgid "%{count} Transactions" msgstr "" @@ -480,7 +480,7 @@ msgid "Copy Txn Hash" msgstr "" #, elixir-format -#: lib/block_scout_web/templates/block/overview.html.eex:59 +#: lib/block_scout_web/templates/block/overview.html.eex:58 msgid "Difficulty" msgstr "" @@ -562,15 +562,15 @@ msgstr "" #, elixir-format #: lib/block_scout_web/templates/block/_tile.html.eex:56 -#: lib/block_scout_web/templates/block/overview.html.eex:109 -#: lib/block_scout_web/templates/block/overview.html.eex:159 +#: lib/block_scout_web/templates/block/overview.html.eex:108 +#: lib/block_scout_web/templates/block/overview.html.eex:158 msgid "Gas Limit" msgstr "" #, elixir-format #: lib/block_scout_web/templates/block/_tile.html.eex:61 -#: lib/block_scout_web/templates/block/overview.html.eex:102 -#: lib/block_scout_web/templates/block/overview.html.eex:153 +#: lib/block_scout_web/templates/block/overview.html.eex:101 +#: lib/block_scout_web/templates/block/overview.html.eex:152 msgid "Gas Used" msgstr "" @@ -657,7 +657,7 @@ msgstr "" #, elixir-format #: lib/block_scout_web/templates/block/_tile.html.eex:38 -#: lib/block_scout_web/templates/block/overview.html.eex:125 +#: lib/block_scout_web/templates/block/overview.html.eex:124 #: lib/block_scout_web/templates/chain/_block.html.eex:15 msgid "Miner" msgstr "" @@ -712,7 +712,7 @@ msgid "Execute" msgstr "" #, elixir-format -#: lib/block_scout_web/templates/block/overview.html.eex:74 +#: lib/block_scout_web/templates/block/overview.html.eex:73 #: lib/block_scout_web/templates/transaction/overview.html.eex:79 msgid "Nonce" msgstr "" @@ -767,7 +767,7 @@ msgid "GET" msgstr "" #, elixir-format -#: lib/block_scout_web/templates/block/overview.html.eex:85 +#: lib/block_scout_web/templates/block/overview.html.eex:84 msgid "Position %{index}" msgstr "" @@ -798,7 +798,7 @@ msgid "Gwei" msgstr "" #, elixir-format -#: lib/block_scout_web/templates/block/overview.html.eex:37 +#: lib/block_scout_web/templates/block/overview.html.eex:36 msgid "Hash" msgstr "" @@ -964,7 +964,7 @@ msgid "Top Accounts - %{subnetwork} Explorer" msgstr "" #, elixir-format -#: lib/block_scout_web/templates/block/overview.html.eex:68 +#: lib/block_scout_web/templates/block/overview.html.eex:67 msgid "Total Difficulty" msgstr "" @@ -1246,7 +1246,7 @@ msgid "Parameters" msgstr "" #, elixir-format -#: lib/block_scout_web/templates/block/overview.html.eex:44 +#: lib/block_scout_web/templates/block/overview.html.eex:43 msgid "Parent Hash" msgstr "" @@ -1404,7 +1404,7 @@ msgid "Show Validator Info" msgstr "" #, elixir-format -#: lib/block_scout_web/templates/block/overview.html.eex:146 +#: lib/block_scout_web/templates/block/overview.html.eex:145 msgid "Block Rewards" msgstr "" @@ -1646,7 +1646,7 @@ msgid "Uncle Reward" msgstr "" #, elixir-format -#: lib/block_scout_web/templates/block/overview.html.eex:81 +#: lib/block_scout_web/templates/block/overview.html.eex:80 #: lib/block_scout_web/templates/layout/_topnav.html.eex:52 msgid "Uncles" msgstr "" From 0944ea5cdcc186f35ed5d89eebf151e7da9f6d32 Mon Sep 17 00:00:00 2001 From: pasqu4le Date: Fri, 20 Sep 2019 18:01:19 +0200 Subject: [PATCH 55/67] Improve speed of nonconsensus data removal Problem: removal of nonconsensus data is too inefficient and as a result blocks are imported too slow. Solution: reformulation of deletion logic for better performance --- .../explorer/chain/import/runner/blocks.ex | 246 ++++++------------ .../chain/import/runner/blocks_test.exs | 12 +- 2 files changed, 91 insertions(+), 167 deletions(-) diff --git a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex index f893df7684..394e21cbaa 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex @@ -46,32 +46,14 @@ defmodule Explorer.Chain.Import.Runner.Blocks do hashes = Enum.map(changes_list, & &1.hash) consensus_block_numbers = consensus_block_numbers(changes_list) - where_invalid_neighbour = where_invalid_neighbour(changes_list) # Enforce ShareLocks tables order (see docs: sharelocks.md) multi - |> Multi.run(:acquire_blocks, fn repo, _ -> - acquire_blocks(repo, hashes, consensus_block_numbers, where_invalid_neighbour) - end) |> Multi.run(:lose_consensus, fn repo, _ -> - lose_consensus(repo, consensus_block_numbers, insert_options) - end) - |> Multi.run(:lose_invalid_neighbour_consensus, fn repo, _ -> - lose_invalid_neighbour_consensus(repo, where_invalid_neighbour, insert_options) - end) - |> Multi.run(:nonconsensus_block_numbers, fn _repo, - %{ - lose_consensus: lost_consensus_blocks, - lose_invalid_neighbour_consensus: lost_consensus_neighbours - } -> - nonconsensus_block_numbers = - (lost_consensus_blocks ++ lost_consensus_neighbours) - |> Enum.sort() - |> Enum.dedup() - - {:ok, nonconsensus_block_numbers} + lose_consensus(repo, hashes, consensus_block_numbers, changes_list, insert_options) end) |> Multi.run(:blocks, fn repo, _ -> + # Note, needs to be executed after `lose_consensus` for lock acquisition insert(repo, changes_list, insert_options) end) |> Multi.run(:uncle_fetched_block_second_degree_relations, fn repo, %{blocks: blocks} when is_list(blocks) -> @@ -101,27 +83,14 @@ defmodule Explorer.Chain.Import.Runner.Blocks do transactions: transactions }) end) - |> Multi.run(:remove_nonconsensus_logs, fn repo, - %{ - nonconsensus_block_numbers: nonconsensus_block_numbers, - fork_transactions: transactions - } -> - remove_nonconsensus_logs(repo, nonconsensus_block_numbers, transactions, insert_options) + |> Multi.run(:remove_nonconsensus_logs, fn repo, %{derive_transaction_forks: transactions} -> + remove_nonconsensus_logs(repo, transactions, insert_options) end) - |> Multi.run(:acquire_internal_transactions, fn repo, - %{ - nonconsensus_block_numbers: nonconsensus_block_numbers, - fork_transactions: transactions - } -> - acquire_internal_transactions(repo, nonconsensus_block_numbers, hashes, transactions) + |> Multi.run(:acquire_internal_transactions, fn repo, %{derive_transaction_forks: transactions} -> + acquire_internal_transactions(repo, hashes, transactions) end) - |> Multi.run(:remove_nonconsensus_internal_transactions, fn repo, - %{ - nonconsensus_block_numbers: - nonconsensus_block_numbers, - fork_transactions: transactions - } -> - remove_nonconsensus_internal_transactions(repo, nonconsensus_block_numbers, transactions, insert_options) + |> Multi.run(:remove_nonconsensus_internal_transactions, fn repo, %{derive_transaction_forks: transactions} -> + remove_nonconsensus_internal_transactions(repo, transactions, insert_options) end) |> Multi.run(:internal_transaction_transaction_block_number, fn repo, _ -> update_internal_transaction_block_number(repo, hashes) @@ -129,9 +98,8 @@ defmodule Explorer.Chain.Import.Runner.Blocks do |> Multi.run(:acquire_contract_address_tokens, fn repo, _ -> acquire_contract_address_tokens(repo, consensus_block_numbers) end) - |> Multi.run(:remove_nonconsensus_token_transfers, fn repo, - %{nonconsensus_block_numbers: nonconsensus_block_numbers} -> - remove_nonconsensus_token_transfers(repo, nonconsensus_block_numbers, insert_options) + |> Multi.run(:remove_nonconsensus_token_transfers, fn repo, %{derive_transaction_forks: transactions} -> + remove_nonconsensus_token_transfers(repo, transactions, insert_options) end) |> Multi.run(:delete_address_token_balances, fn repo, _ -> delete_address_token_balances(repo, consensus_block_numbers, insert_options) @@ -159,22 +127,6 @@ defmodule Explorer.Chain.Import.Runner.Blocks do @impl Runner def timeout, do: @timeout - defp acquire_blocks(repo, hashes, consensus_block_numbers, where_invalid_neighbour) do - query = - from( - block in where_invalid_neighbour, - or_where: block.number in ^consensus_block_numbers, - or_where: block.hash in ^hashes, - select: block.hash, - # Enforce Block ShareLocks order (see docs: sharelocks.md) - order_by: [asc: block.hash], - lock: "FOR UPDATE" - ) - - blocks = repo.all(query) - {:ok, blocks} - end - defp acquire_contract_address_tokens(repo, consensus_block_numbers) do query = from(address_current_token_balance in Address.CurrentTokenBalance, @@ -187,15 +139,12 @@ defmodule Explorer.Chain.Import.Runner.Blocks do Tokens.acquire_contract_address_tokens(repo, contract_address_hashes) end - defp acquire_internal_transactions(repo, nonconsensus_block_numbers, hashes, forked_transactions) do - forked_transaction_hashes = Enum.map(forked_transactions, & &1.hash) - + defp acquire_internal_transactions(repo, hashes, forked_transaction_hashes) do query = from(internal_transaction in InternalTransaction, join: transaction in Transaction, on: internal_transaction.transaction_hash == transaction.hash, - where: transaction.block_number in ^nonconsensus_block_numbers, - or_where: transaction.block_hash in ^hashes, + where: transaction.block_hash in ^hashes, or_where: transaction.hash in ^forked_transaction_hashes, select: {internal_transaction.transaction_hash, internal_transaction.index}, # Enforce InternalTransaction ShareLocks order (see docs: sharelocks.md) @@ -229,14 +178,11 @@ defmodule Explorer.Chain.Import.Runner.Blocks do lock: "FOR UPDATE" ) - transactions = repo.all(query) - - hashes = Enum.map(transactions, & &1.hash) - update_query = from( t in Transaction, - where: t.hash in ^hashes, + join: s in subquery(query), + on: t.hash == s.hash, update: [ set: [ block_hash: nil, @@ -250,17 +196,19 @@ defmodule Explorer.Chain.Import.Runner.Blocks do updated_at: ^updated_at ] ], - select: t.hash + select: %{ + block_hash: s.block_hash, + index: s.index, + hash: s.hash + } ) - try do - {_num, _res} = repo.update_all(update_query, [], timeout: timeout) + {_num, transactions} = repo.update_all(update_query, [], timeout: timeout) - {:ok, transactions} - rescue - postgrex_error in Postgrex.Error -> - {:error, %{exception: postgrex_error}} - end + {:ok, transactions} + rescue + postgrex_error in Postgrex.Error -> + {:error, %{exception: postgrex_error}} end defp derive_transaction_forks(%{ @@ -283,7 +231,7 @@ defmodule Explorer.Chain.Import.Runner.Blocks do # Enforce Fork ShareLocks order (see docs: sharelocks.md) |> Enum.sort_by(&{&1.uncle_hash, &1.index}) - {_total, result} = + {_total, forked_transaction} = repo.insert_all( Transaction.Fork, transaction_forks, @@ -294,11 +242,11 @@ defmodule Explorer.Chain.Import.Runner.Blocks do update: [set: [hash: fragment("EXCLUDED.hash")]], where: fragment("EXCLUDED.hash <> ?", transaction_fork.hash) ), - returning: [:uncle_hash, :hash], + returning: [:hash], timeout: timeout ) - {:ok, result} + {:ok, Enum.map(forked_transaction, & &1.hash)} end @spec insert(Repo.t(), [map()], %{ @@ -364,47 +312,48 @@ defmodule Explorer.Chain.Import.Runner.Blocks do |> Enum.map(& &1.number) end - defp lose_consensus(_, [], _), do: {:ok, []} - - defp lose_consensus(repo, consensus_block_number, %{timeout: timeout, timestamps: %{updated_at: updated_at}}) - when is_list(consensus_block_number) do - # ShareLocks order already enforced by `acquire_blocks` (see docs: sharelocks.md) - {_, result} = - repo.update_all( - from(block in Block, where: block.number in ^consensus_block_number, select: block.number), - [set: [consensus: false, updated_at: updated_at]], - timeout: timeout - ) - - {:ok, result} - rescue - postgrex_error in Postgrex.Error -> - {:error, %{exception: postgrex_error, consensus_block_numbers: consensus_block_number}} - end - - defp lose_invalid_neighbour_consensus(repo, where_invalid_neighbour, %{ + defp lose_consensus(repo, hashes, consensus_block_numbers, changes_list, %{ timeout: timeout, timestamps: %{updated_at: updated_at} }) do - # ShareLocks order already enforced by `acquire_blocks` (see docs: sharelocks.md) - {_, result} = + acquire_query = + from( + block in where_invalid_neighbour(changes_list), + or_where: block.number in ^consensus_block_numbers, + # we also need to acquire blocks that will be upserted here, for ordering + or_where: block.hash in ^hashes, + select: block.hash, + # Enforce Block ShareLocks order (see docs: sharelocks.md) + order_by: [asc: block.hash], + lock: "FOR UPDATE" + ) + + {_, removed_consensus_block_hashes} = repo.update_all( - from(block in where_invalid_neighbour, select: block.number), + from( + block in Block, + join: s in subquery(acquire_query), + on: block.hash == s.hash, + # we don't want to remove consensus from blocks that will be upserted + where: block.hash not in ^hashes, + select: block.hash + ), [set: [consensus: false, updated_at: updated_at]], timeout: timeout ) - {:ok, result} + {:ok, removed_consensus_block_hashes} rescue postgrex_error in Postgrex.Error -> - {:error, %{exception: postgrex_error, where_invalid_neighbour: where_invalid_neighbour}} + {:error, %{exception: postgrex_error, consensus_block_numbers: consensus_block_numbers}} end - defp remove_nonconsensus_token_transfers(repo, nonconsensus_block_numbers, %{timeout: timeout}) do + defp remove_nonconsensus_token_transfers(repo, forked_transaction_hashes, %{timeout: timeout}) do ordered_token_transfers = - from(token_transfer in TokenTransfer, - where: token_transfer.block_number in ^nonconsensus_block_numbers, - select: map(token_transfer, [:transaction_hash, :log_index]), + from( + token_transfer in TokenTransfer, + where: token_transfer.transaction_hash in ^forked_transaction_hashes, + select: token_transfer.transaction_hash, # Enforce TokenTransfer ShareLocks order (see docs: sharelocks.md) order_by: [ token_transfer.transaction_hash, @@ -417,91 +366,60 @@ defmodule Explorer.Chain.Import.Runner.Blocks do from(token_transfer in TokenTransfer, select: map(token_transfer, [:transaction_hash, :log_index]), inner_join: ordered_token_transfer in subquery(ordered_token_transfers), - on: - ordered_token_transfer.transaction_hash == - token_transfer.transaction_hash and - ordered_token_transfer.log_index == token_transfer.log_index + on: ordered_token_transfer.transaction_hash == token_transfer.transaction_hash ) - try do - {_count, deleted_token_transfers} = repo.delete_all(query, timeout: timeout) + {_count, deleted_token_transfers} = repo.delete_all(query, timeout: timeout) - {:ok, deleted_token_transfers} - rescue - postgrex_error in Postgrex.Error -> - {:error, %{exception: postgrex_error, block_numbers: nonconsensus_block_numbers}} - end + {:ok, deleted_token_transfers} + rescue + postgrex_error in Postgrex.Error -> + {:error, %{exception: postgrex_error, transactions: forked_transaction_hashes}} end - defp remove_nonconsensus_internal_transactions(repo, nonconsensus_block_numbers, forked_transactions, %{ - timeout: timeout - }) do - forked_transaction_hashes = Enum.map(forked_transactions, & &1.hash) - - transaction_query = - from(transaction in Transaction, - where: transaction.block_number in ^nonconsensus_block_numbers, - or_where: transaction.hash in ^forked_transaction_hashes, - select: map(transaction, [:hash]) - ) - + defp remove_nonconsensus_internal_transactions(repo, forked_transaction_hashes, %{timeout: timeout}) do query = from(internal_transaction in InternalTransaction, - inner_join: transaction in subquery(transaction_query), - on: internal_transaction.transaction_hash == transaction.hash, + where: internal_transaction.transaction_hash in ^forked_transaction_hashes, select: map(internal_transaction, [:transaction_hash, :index]) ) - try do - # ShareLocks order already enforced by `acquire_internal_transactions` (see docs: sharelocks.md) - {_count, deleted_internal_transactions} = repo.delete_all(query, timeout: timeout) + # ShareLocks order already enforced by `acquire_internal_transactions` (see docs: sharelocks.md) + {_count, deleted_internal_transactions} = repo.delete_all(query, timeout: timeout) - {:ok, deleted_internal_transactions} - rescue - postgrex_error in Postgrex.Error -> - {:error, %{exception: postgrex_error, block_numbers: nonconsensus_block_numbers}} - end + {:ok, deleted_internal_transactions} + rescue + postgrex_error in Postgrex.Error -> + {:error, %{exception: postgrex_error, transactions: forked_transaction_hashes}} end - defp remove_nonconsensus_logs(repo, nonconsensus_block_numbers, forked_transactions, %{timeout: timeout}) do - forked_transaction_hashes = Enum.map(forked_transactions, & &1.hash) - - transaction_query = - from(transaction in Transaction, - where: transaction.block_number in ^nonconsensus_block_numbers, - or_where: transaction.hash in ^forked_transaction_hashes, - select: map(transaction, [:hash]), - order_by: transaction.hash - ) - + defp remove_nonconsensus_logs(repo, forked_transaction_hashes, %{timeout: timeout}) do ordered_logs = - from(log in Log, - inner_join: transaction in subquery(transaction_query), - on: log.transaction_hash == transaction.hash, - select: map(log, [:transaction_hash, :index]), + from( + log in Log, + where: log.transaction_hash in ^forked_transaction_hashes, + select: log.transaction_hash, # Enforce Log ShareLocks order (see docs: sharelocks.md) order_by: [ log.transaction_hash, log.index ], - lock: "FOR UPDATE OF l0" + lock: "FOR UPDATE" ) query = from(log in Log, select: map(log, [:transaction_hash, :index]), inner_join: ordered_log in subquery(ordered_logs), - on: ordered_log.transaction_hash == log.transaction_hash and ordered_log.index == log.index + on: ordered_log.transaction_hash == log.transaction_hash ) - try do - {_count, deleted_logs} = repo.delete_all(query, timeout: timeout) + {_count, deleted_logs} = repo.delete_all(query, timeout: timeout) - {:ok, deleted_logs} - rescue - postgrex_error in Postgrex.Error -> - {:error, %{exception: postgrex_error, block_numbers: nonconsensus_block_numbers}} - end + {:ok, deleted_logs} + rescue + postgrex_error in Postgrex.Error -> + {:error, %{exception: postgrex_error, transactions: forked_transaction_hashes}} end defp delete_address_token_balances(_, [], _), do: {:ok, []} diff --git a/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs b/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs index 370fd65021..3cbd7f4834 100644 --- a/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs +++ b/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs @@ -117,10 +117,12 @@ defmodule Explorer.Chain.Import.Runner.BlocksTest do test "remove_nonconsensus_token_transfers deletes token transfer rows with matching block number when new consensus block is inserted", %{consensus_block: %{number: block_number} = block, options: options} do - insert(:block, number: block_number, consensus: true) + consensus_block = insert(:block, number: block_number, consensus: true) + + transaction = insert(:transaction) |> with_block(consensus_block) %TokenTransfer{transaction_hash: transaction_hash, log_index: log_index} = - insert(:token_transfer, block_number: block_number, transaction: insert(:transaction)) + insert(:token_transfer, block_number: block_number, transaction: transaction) assert count(TokenTransfer) == 1 @@ -136,7 +138,11 @@ defmodule Explorer.Chain.Import.Runner.BlocksTest do test "remove_nonconsensus_token_transfers does not delete token transfer rows with matching block number when new consensus block wasn't inserted", %{consensus_block: %{number: block_number} = block, options: options} do - insert(:token_transfer, block_number: block_number, transaction: insert(:transaction)) + consensus_block = insert(:block, number: block_number, consensus: true) + + transaction = insert(:transaction) |> with_block(consensus_block) + + insert(:token_transfer, block_number: block_number, transaction: transaction) count = 1 From bc47fe957e77116f342fa1db37d5a86f1b77abe0 Mon Sep 17 00:00:00 2001 From: pasqu4le Date: Fri, 20 Sep 2019 18:14:05 +0200 Subject: [PATCH 56/67] changelog entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6402a0acd..a525fc5951 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## Current ### Features +- [#2717](https://github.com/poanetwork/blockscout/pull/2717) - Improve speed of nonconsensus data removal - [#2679](https://github.com/poanetwork/blockscout/pull/2679) - added fixed height for card chain blocks and card chain transactions - [#2678](https://github.com/poanetwork/blockscout/pull/2678) - fixed dashboard banner height bug - [#2672](https://github.com/poanetwork/blockscout/pull/2672) - added new theme for xUSDT From a4a6d93834de12b1829836ec23ad08d9d71657eb Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Mon, 23 Sep 2019 12:04:19 +0300 Subject: [PATCH 57/67] get rid of json schema warnings --- apps/block_scout_web/mix.exs | 2 +- mix.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/block_scout_web/mix.exs b/apps/block_scout_web/mix.exs index de55cb0eb7..029819a202 100644 --- a/apps/block_scout_web/mix.exs +++ b/apps/block_scout_web/mix.exs @@ -131,7 +131,7 @@ defmodule BlockScoutWeb.Mixfile do # `:cowboy` `~> 2.0` and Phoenix 1.4 compatibility {:wobserver, "~> 0.2.0", github: "poanetwork/wobserver", branch: "support-https"}, {:phoenix_form_awesomplete, "~> 0.1.4"}, - {:ex_json_schema, "~> 0.6.1"} + {:ex_json_schema, "~> 0.6.2"} ] end diff --git a/mix.lock b/mix.lock index e0d75af01a..87f6e4e2f0 100644 --- a/mix.lock +++ b/mix.lock @@ -39,7 +39,7 @@ "ex_cldr_numbers": {:hex, :ex_cldr_numbers, "2.6.4", "5b1ac8451f889576bb29dee70412de1170974298727ab944aa4d17e91bdd3472", [:mix], [{:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:ex_cldr, "~> 2.6", [hex: :ex_cldr, repo: "hexpm", optional: false]}, {:ex_cldr_currencies, "~> 2.3", [hex: :ex_cldr_currencies, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, "ex_cldr_units": {:hex, :ex_cldr_units, "2.5.1", "0e65067a22a7c5146266c313d6333c2700868c32aa6d536f47c6c0d84aac3ac1", [:mix], [{:ex_cldr, "~> 2.6", [hex: :ex_cldr, repo: "hexpm", optional: false]}, {:ex_cldr_lists, "~> 2.2", [hex: :ex_cldr_lists, repo: "hexpm", optional: false]}, {:ex_cldr_numbers, "~> 2.6", [hex: :ex_cldr_numbers, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, "ex_doc": {:hex, :ex_doc, "0.19.3", "3c7b0f02851f5fc13b040e8e925051452e41248f685e40250d7e40b07b9f8c10", [:mix], [{:earmark, "~> 1.2", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.10", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm"}, - "ex_json_schema": {:hex, :ex_json_schema, "0.6.1", "b57c0588385b8262b80f19d33d9b9b71fcd60d247691abf2635b57a03ec0ad44", [:mix], [], "hexpm"}, + "ex_json_schema": {:hex, :ex_json_schema, "0.6.2", "de23d80478215987469c81688208fe0ff440ee0e0e6ae2268fcadbb2ff35df9d", [:mix], [], "hexpm"}, "ex_machina": {:hex, :ex_machina, "2.3.0", "92a5ad0a8b10ea6314b876a99c8c9e3f25f4dde71a2a835845b136b9adaf199a", [:mix], [{:ecto, "~> 2.2 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}], "hexpm"}, "ex_rlp": {:hex, :ex_rlp, "0.5.2", "7f4ce7bd55e543c054ce6d49629b01e9833c3462e3d547952be89865f39f2c58", [:mix], [], "hexpm"}, "ex_utils": {:hex, :ex_utils, "0.1.7", "2c133e0bcdc49a858cf8dacf893308ebc05bc5fba501dc3d2935e65365ec0bf3", [:mix], [], "hexpm"}, From 2f584ef19a571650545ad74e1e58afa36ccf94d3 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Mon, 23 Sep 2019 12:08:08 +0300 Subject: [PATCH 58/67] add CHANGELOG entry --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6402a0acd..5a40460247 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,8 @@ ### Chore +- [#2723](https://github.com/poanetwork/blockscout/pull/2723) - get rid of ex_json_schema warnings + ## 2.0.4-beta From a94d2fb37e054690895a0e4f888da6af660ba0b0 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Mon, 23 Sep 2019 13:38:06 +0300 Subject: [PATCH 59/67] fix ci --- .circleci/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index b3abd48786..ba922ddfff 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -39,6 +39,9 @@ jobs: - run: mix deps.get + - run: + command: sed -i '68,68 s/^/%/' ./deps/hackney/src/hackney_ssl.erl + - restore_cache: keys: - v7-npm-install-{{ .Branch }}-{{ checksum "apps/block_scout_web/assets/package-lock.json" }} From 765987979ddaf6fa215ffe477d6315fe42752145 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Mon, 23 Sep 2019 13:48:14 +0300 Subject: [PATCH 60/67] bump cache version --- .circleci/config.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ba922ddfff..a2b372d416 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -33,9 +33,9 @@ jobs: - restore_cache: keys: - - v7-mix-compile-{{ checksum "OTP_VERSION.lock" }}-{{ checksum "ELIXIR_VERSION.lock" }}-{{ checksum "mix.lock" }} - - v7-mix-compile-{{ checksum "OTP_VERSION.lock" }}-{{ checksum "ELIXIR_VERSION.lock" }}-{{ checksum "mix.exs" }} - - v7-mix-compile-{{ checksum "OTP_VERSION.lock" }}-{{ checksum "ELIXIR_VERSION.lock" }} + - v8-mix-compile-{{ checksum "OTP_VERSION.lock" }}-{{ checksum "ELIXIR_VERSION.lock" }}-{{ checksum "mix.lock" }} + - v8-mix-compile-{{ checksum "OTP_VERSION.lock" }}-{{ checksum "ELIXIR_VERSION.lock" }}-{{ checksum "mix.exs" }} + - v8-mix-compile-{{ checksum "OTP_VERSION.lock" }}-{{ checksum "ELIXIR_VERSION.lock" }} - run: mix deps.get @@ -86,17 +86,17 @@ jobs: # `deps` needs to be cached with `_build` because `_build` will symlink into `deps` - save_cache: - key: v7-mix-compile-{{ checksum "OTP_VERSION.lock" }}-{{ checksum "ELIXIR_VERSION.lock" }}-{{ checksum "mix.lock" }} + key: v8-mix-compile-{{ checksum "OTP_VERSION.lock" }}-{{ checksum "ELIXIR_VERSION.lock" }}-{{ checksum "mix.lock" }} paths: - deps - _build - save_cache: - key: v7-mix-compile-{{ checksum "OTP_VERSION.lock" }}-{{ checksum "ELIXIR_VERSION.lock" }}-{{ checksum "mix.exs" }} + key: v8-mix-compile-{{ checksum "OTP_VERSION.lock" }}-{{ checksum "ELIXIR_VERSION.lock" }}-{{ checksum "mix.exs" }} paths: - deps - _build - save_cache: - key: v7-mix-compile-{{ checksum "OTP_VERSION.lock" }}-{{ checksum "ELIXIR_VERSION.lock" }} + key: v8-mix-compile-{{ checksum "OTP_VERSION.lock" }}-{{ checksum "ELIXIR_VERSION.lock" }} paths: - deps - _build From f19814cade66a9f9f192b4ec24d94909d2e63b3c Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Mon, 23 Sep 2019 14:01:22 +0300 Subject: [PATCH 61/67] add CHANGELOG entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6402a0acd..7c0c50d0f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ - [#2619](https://github.com/poanetwork/blockscout/pull/2619) - Enforce DB transaction's order to prevent deadlocks ### Chore +- [#2724](https://github.com/poanetwork/blockscout/pull/2724) - fix ci by commenting a line in hackney library ## 2.0.4-beta From 455d6d647f9e50807e3137c4673dc9a0b5bad204 Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Fri, 20 Sep 2019 15:29:41 +0300 Subject: [PATCH 62/67] Addresses counter --- CHANGELOG.md | 1 + .../controllers/address_controller.ex | 2 +- .../controllers/chain_controller.ex | 2 +- .../lib/block_scout_web/notifier.ex | 2 +- .../channels/address_channel_test.exs | 16 +-- .../controllers/address_controller_test.exs | 10 +- .../api/rpc/address_controller_test.exs | 4 +- .../api/rpc/eth_controller_test.exs | 4 +- .../controllers/chain_controller_test.exs | 8 +- .../features/viewing_addresses_test.exs | 6 +- .../features/viewing_app_test.exs | 6 +- .../features/viewing_chain_test.exs | 34 ++--- apps/explorer/config/config.exs | 5 + apps/explorer/config/test.exs | 2 + apps/explorer/lib/explorer/application.ex | 1 + apps/explorer/lib/explorer/chain.ex | 23 +++- apps/explorer/lib/explorer/chain/address.ex | 10 ++ .../explorer/counters/addresses_counter.ex | 125 ++++++++++++++++++ apps/explorer/test/explorer/chain_test.exs | 17 +++ .../counters/addresses_counter_test.exs | 16 +++ 20 files changed, 246 insertions(+), 48 deletions(-) create mode 100644 apps/explorer/lib/explorer/counters/addresses_counter.ex create mode 100644 apps/explorer/test/explorer/counters/addresses_counter_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index b6402a0acd..5fbc67722c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ - [#2663](https://github.com/poanetwork/blockscout/pull/2663) - Fetch address counters in parallel ### Fixes +- [#2718](https://github.com/poanetwork/blockscout/pull/2718) - Include all addresses taking part in transactions in wallets' addresses counter - [#2709](https://github.com/poanetwork/blockscout/pull/2709) - Fix stuck label and value for uncle block height - [#2707](https://github.com/poanetwork/blockscout/pull/2707) - fix for dashboard banner chart legend items - [#2706](https://github.com/poanetwork/blockscout/pull/2706) - fix empty total_supply in coin gecko response diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/address_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/address_controller.ex index 5916359281..e2cb1f6db7 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/address_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/address_controller.ex @@ -61,7 +61,7 @@ defmodule BlockScoutWeb.AddressController do def index(conn, _params) do render(conn, "index.html", current_path: current_path(conn), - address_count: Chain.count_addresses_with_balance_from_cache() + address_count: Chain.count_addresses_from_cache() ) end diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/chain_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/chain_controller.ex index ba5d547cf0..936d3ab0a0 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/chain_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/chain_controller.ex @@ -28,7 +28,7 @@ defmodule BlockScoutWeb.ChainController do render( conn, "show.html", - address_count: Chain.count_addresses_with_balance_from_cache(), + address_count: Chain.count_addresses_from_cache(), average_block_time: AverageBlockTime.average_block_time(), exchange_rate: exchange_rate, chart_data_path: market_history_chart_path(conn, :show), diff --git a/apps/block_scout_web/lib/block_scout_web/notifier.ex b/apps/block_scout_web/lib/block_scout_web/notifier.ex index 7c07c8524e..5955ca02d4 100644 --- a/apps/block_scout_web/lib/block_scout_web/notifier.ex +++ b/apps/block_scout_web/lib/block_scout_web/notifier.ex @@ -14,7 +14,7 @@ defmodule BlockScoutWeb.Notifier do alias Phoenix.View def handle_event({:chain_event, :addresses, type, addresses}) when type in [:realtime, :on_demand] do - Endpoint.broadcast("addresses:new_address", "count", %{count: Chain.count_addresses_with_balance_from_cache()}) + Endpoint.broadcast("addresses:new_address", "count", %{count: Chain.count_addresses_from_cache()}) addresses |> Stream.reject(fn %Address{fetched_coin_balance: fetched_coin_balance} -> is_nil(fetched_coin_balance) end) diff --git a/apps/block_scout_web/test/block_scout_web/channels/address_channel_test.exs b/apps/block_scout_web/test/block_scout_web/channels/address_channel_test.exs index 49d643fe27..357a73eba8 100644 --- a/apps/block_scout_web/test/block_scout_web/channels/address_channel_test.exs +++ b/apps/block_scout_web/test/block_scout_web/channels/address_channel_test.exs @@ -1,11 +1,11 @@ defmodule BlockScoutWeb.AddressChannelTest do use BlockScoutWeb.ChannelCase, - # ETS tables are shared in `Explorer.Counters.AddressesWithBalanceCounter` + # ETS tables are shared in `Explorer.Counters.AddressesCounter` async: false alias BlockScoutWeb.UserSocket alias BlockScoutWeb.Notifier - alias Explorer.Counters.AddressesWithBalanceCounter + alias Explorer.Counters.AddressesCounter test "subscribed user is notified of new_address count event" do topic = "addresses:new_address" @@ -13,8 +13,8 @@ defmodule BlockScoutWeb.AddressChannelTest do address = insert(:address) - start_supervised!(AddressesWithBalanceCounter) - AddressesWithBalanceCounter.consolidate() + start_supervised!(AddressesCounter) + AddressesCounter.consolidate() Notifier.handle_event({:chain_event, :addresses, :realtime, [address]}) @@ -55,8 +55,8 @@ defmodule BlockScoutWeb.AddressChannelTest do test "notified of balance_update for matching address", %{address: address, topic: topic} do address_with_balance = %{address | fetched_coin_balance: 1} - start_supervised!(AddressesWithBalanceCounter) - AddressesWithBalanceCounter.consolidate() + start_supervised!(AddressesCounter) + AddressesCounter.consolidate() Notifier.handle_event({:chain_event, :addresses, :realtime, [address_with_balance]}) @@ -67,8 +67,8 @@ defmodule BlockScoutWeb.AddressChannelTest do end test "not notified of balance_update if fetched_coin_balance is nil", %{address: address} do - start_supervised!(AddressesWithBalanceCounter) - AddressesWithBalanceCounter.consolidate() + start_supervised!(AddressesCounter) + AddressesCounter.consolidate() Notifier.handle_event({:chain_event, :addresses, :realtime, [address]}) diff --git a/apps/block_scout_web/test/block_scout_web/controllers/address_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/address_controller_test.exs index 7eb0545188..baae81bc1f 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/address_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/address_controller_test.exs @@ -3,7 +3,7 @@ defmodule BlockScoutWeb.AddressControllerTest do # ETS tables are shared in `Explorer.Counters.*` async: false - alias Explorer.Counters.AddressesWithBalanceCounter + alias Explorer.Counters.AddressesCounter describe "GET index/2" do test "returns top addresses", %{conn: conn} do @@ -12,8 +12,8 @@ defmodule BlockScoutWeb.AddressControllerTest do |> Enum.map(&insert(:address, fetched_coin_balance: &1)) |> Enum.map(& &1.hash) - start_supervised!(AddressesWithBalanceCounter) - AddressesWithBalanceCounter.consolidate() + start_supervised!(AddressesCounter) + AddressesCounter.consolidate() conn = get(conn, address_path(conn, :index, %{type: "JSON"})) {:ok, %{"items" => items}} = Poison.decode(conn.resp_body) @@ -25,8 +25,8 @@ defmodule BlockScoutWeb.AddressControllerTest do address = insert(:address, fetched_coin_balance: 1) insert(:address_name, address: address, primary: true, name: "POA Wallet") - start_supervised!(AddressesWithBalanceCounter) - AddressesWithBalanceCounter.consolidate() + start_supervised!(AddressesCounter) + AddressesCounter.consolidate() conn = get(conn, address_path(conn, :index, %{type: "JSON"})) diff --git a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/address_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/address_controller_test.exs index 96716f0935..7e321ac4cf 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/address_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/address_controller_test.exs @@ -6,7 +6,7 @@ defmodule BlockScoutWeb.API.RPC.AddressControllerTest do alias BlockScoutWeb.API.RPC.AddressController alias Explorer.Chain alias Explorer.Chain.{Events.Subscriber, Transaction, Wei} - alias Explorer.Counters.{AddressesWithBalanceCounter, AverageBlockTime} + alias Explorer.Counters.{AddressesCounter, AverageBlockTime} alias Indexer.Fetcher.CoinBalanceOnDemand alias Explorer.Repo @@ -22,7 +22,7 @@ defmodule BlockScoutWeb.API.RPC.AddressControllerTest do start_supervised!({Task.Supervisor, name: Indexer.TaskSupervisor}) start_supervised!(AverageBlockTime) start_supervised!({CoinBalanceOnDemand, [mocked_json_rpc_named_arguments, [name: CoinBalanceOnDemand]]}) - start_supervised!(AddressesWithBalanceCounter) + start_supervised!(AddressesCounter) Application.put_env(:explorer, AverageBlockTime, enabled: true) diff --git a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/eth_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/eth_controller_test.exs index 4b43909ac4..5ab050aefc 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/eth_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/eth_controller_test.exs @@ -1,7 +1,7 @@ defmodule BlockScoutWeb.API.RPC.EthControllerTest do use BlockScoutWeb.ConnCase, async: false - alias Explorer.Counters.{AddressesWithBalanceCounter, AverageBlockTime} + alias Explorer.Counters.{AddressesCounter, AverageBlockTime} alias Explorer.Repo alias Indexer.Fetcher.CoinBalanceOnDemand @@ -14,7 +14,7 @@ defmodule BlockScoutWeb.API.RPC.EthControllerTest do start_supervised!({Task.Supervisor, name: Indexer.TaskSupervisor}) start_supervised!(AverageBlockTime) start_supervised!({CoinBalanceOnDemand, [mocked_json_rpc_named_arguments, [name: CoinBalanceOnDemand]]}) - start_supervised!(AddressesWithBalanceCounter) + start_supervised!(AddressesCounter) Application.put_env(:explorer, AverageBlockTime, enabled: true) diff --git a/apps/block_scout_web/test/block_scout_web/controllers/chain_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/chain_controller_test.exs index fa82bb5bab..def232741c 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/chain_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/chain_controller_test.exs @@ -1,18 +1,18 @@ defmodule BlockScoutWeb.ChainControllerTest do use BlockScoutWeb.ConnCase, - # ETS table is shared in `Explorer.Counters.AddressesWithBalanceCounter` + # ETS table is shared in `Explorer.Counters.AddressesCounter` async: false import BlockScoutWeb.WebRouter.Helpers, only: [chain_path: 2, block_path: 3, transaction_path: 3, address_path: 3] alias Explorer.Chain.Block - alias Explorer.Counters.AddressesWithBalanceCounter + alias Explorer.Counters.AddressesCounter setup do Supervisor.terminate_child(Explorer.Supervisor, Explorer.Chain.Cache.Blocks.child_id()) Supervisor.restart_child(Explorer.Supervisor, Explorer.Chain.Cache.Blocks.child_id()) - start_supervised!(AddressesWithBalanceCounter) - AddressesWithBalanceCounter.consolidate() + start_supervised!(AddressesCounter) + AddressesCounter.consolidate() :ok end diff --git a/apps/block_scout_web/test/block_scout_web/features/viewing_addresses_test.exs b/apps/block_scout_web/test/block_scout_web/features/viewing_addresses_test.exs index b5f17afc49..bb223077a2 100644 --- a/apps/block_scout_web/test/block_scout_web/features/viewing_addresses_test.exs +++ b/apps/block_scout_web/test/block_scout_web/features/viewing_addresses_test.exs @@ -3,7 +3,7 @@ defmodule BlockScoutWeb.ViewingAddressesTest do # Because ETS tables is shared for `Explorer.Counters.*` async: false - alias Explorer.Counters.AddressesWithBalanceCounter + alias Explorer.Counters.AddressesCounter alias BlockScoutWeb.{AddressPage, AddressView, Notifier} setup do @@ -58,8 +58,8 @@ defmodule BlockScoutWeb.ViewingAddressesTest do [first_address | _] = addresses [last_address | _] = Enum.reverse(addresses) - start_supervised!(AddressesWithBalanceCounter) - AddressesWithBalanceCounter.consolidate() + start_supervised!(AddressesCounter) + AddressesCounter.consolidate() session |> AddressPage.visit_page() diff --git a/apps/block_scout_web/test/block_scout_web/features/viewing_app_test.exs b/apps/block_scout_web/test/block_scout_web/features/viewing_app_test.exs index 27742d31eb..ade1f29b15 100644 --- a/apps/block_scout_web/test/block_scout_web/features/viewing_app_test.exs +++ b/apps/block_scout_web/test/block_scout_web/features/viewing_app_test.exs @@ -5,11 +5,11 @@ defmodule BlockScoutWeb.ViewingAppTest do alias BlockScoutWeb.AppPage alias BlockScoutWeb.Counters.BlocksIndexedCounter - alias Explorer.Counters.AddressesWithBalanceCounter + alias Explorer.Counters.AddressesCounter setup do - start_supervised!(AddressesWithBalanceCounter) - AddressesWithBalanceCounter.consolidate() + start_supervised!(AddressesCounter) + AddressesCounter.consolidate() :ok end diff --git a/apps/block_scout_web/test/block_scout_web/features/viewing_chain_test.exs b/apps/block_scout_web/test/block_scout_web/features/viewing_chain_test.exs index 27b4dbb329..367e8b8b7c 100644 --- a/apps/block_scout_web/test/block_scout_web/features/viewing_chain_test.exs +++ b/apps/block_scout_web/test/block_scout_web/features/viewing_chain_test.exs @@ -7,7 +7,7 @@ defmodule BlockScoutWeb.ViewingChainTest do alias BlockScoutWeb.{AddressPage, BlockPage, ChainPage, TransactionPage} alias Explorer.Chain.Block - alias Explorer.Counters.AddressesWithBalanceCounter + alias Explorer.Counters.AddressesCounter setup do Supervisor.terminate_child(Explorer.Supervisor, Explorer.Chain.Cache.Blocks.child_id()) @@ -35,8 +35,8 @@ defmodule BlockScoutWeb.ViewingChainTest do test "search for address", %{session: session} do address = insert(:address) - start_supervised!(AddressesWithBalanceCounter) - AddressesWithBalanceCounter.consolidate() + start_supervised!(AddressesCounter) + AddressesCounter.consolidate() session |> ChainPage.visit_page() @@ -49,8 +49,8 @@ defmodule BlockScoutWeb.ViewingChainTest do test "search for blocks from chain page", %{session: session} do block = insert(:block, number: 6) - start_supervised!(AddressesWithBalanceCounter) - AddressesWithBalanceCounter.consolidate() + start_supervised!(AddressesCounter) + AddressesCounter.consolidate() session |> ChainPage.visit_page() @@ -59,8 +59,8 @@ defmodule BlockScoutWeb.ViewingChainTest do end test "blocks list", %{session: session} do - start_supervised!(AddressesWithBalanceCounter) - AddressesWithBalanceCounter.consolidate() + start_supervised!(AddressesCounter) + AddressesCounter.consolidate() session |> ChainPage.visit_page() @@ -70,8 +70,8 @@ defmodule BlockScoutWeb.ViewingChainTest do test "inserts place holder blocks on render for out of order blocks", %{session: session} do insert(:block, number: 409) - start_supervised!(AddressesWithBalanceCounter) - AddressesWithBalanceCounter.consolidate() + start_supervised!(AddressesCounter) + AddressesCounter.consolidate() session |> ChainPage.visit_page() @@ -84,8 +84,8 @@ defmodule BlockScoutWeb.ViewingChainTest do test "search for transactions", %{session: session} do transaction = insert(:transaction) - start_supervised!(AddressesWithBalanceCounter) - AddressesWithBalanceCounter.consolidate() + start_supervised!(AddressesCounter) + AddressesCounter.consolidate() session |> ChainPage.visit_page() @@ -94,8 +94,8 @@ defmodule BlockScoutWeb.ViewingChainTest do end test "transactions list", %{session: session} do - start_supervised!(AddressesWithBalanceCounter) - AddressesWithBalanceCounter.consolidate() + start_supervised!(AddressesCounter) + AddressesCounter.consolidate() session |> ChainPage.visit_page() @@ -111,8 +111,8 @@ defmodule BlockScoutWeb.ViewingChainTest do |> with_contract_creation(contract_address) |> with_block(block) - start_supervised!(AddressesWithBalanceCounter) - AddressesWithBalanceCounter.consolidate() + start_supervised!(AddressesCounter) + AddressesCounter.consolidate() session |> ChainPage.visit_page() @@ -138,8 +138,8 @@ defmodule BlockScoutWeb.ViewingChainTest do token_contract_address: contract_token_address ) - start_supervised!(AddressesWithBalanceCounter) - AddressesWithBalanceCounter.consolidate() + start_supervised!(AddressesCounter) + AddressesCounter.consolidate() session |> ChainPage.visit_page() diff --git a/apps/explorer/config/config.exs b/apps/explorer/config/config.exs index 48d9b55610..0cf1a05b53 100644 --- a/apps/explorer/config/config.exs +++ b/apps/explorer/config/config.exs @@ -47,6 +47,11 @@ balances_update_interval = end config :explorer, Explorer.Counters.AddressesWithBalanceCounter, + enabled: false, + enable_consolidation: true, + update_interval_in_seconds: balances_update_interval || 30 * 60 + +config :explorer, Explorer.Counters.AddressesCounter, enabled: true, enable_consolidation: true, update_interval_in_seconds: balances_update_interval || 30 * 60 diff --git a/apps/explorer/config/test.exs b/apps/explorer/config/test.exs index d91f4dbdfc..8530014cd8 100644 --- a/apps/explorer/config/test.exs +++ b/apps/explorer/config/test.exs @@ -21,6 +21,8 @@ config :explorer, Explorer.Counters.AverageBlockTime, enabled: false config :explorer, Explorer.Counters.AddressesWithBalanceCounter, enabled: false, enable_consolidation: false +config :explorer, Explorer.Counters.AddressesCounter, enabled: false, enable_consolidation: false + config :explorer, Explorer.Market.History.Cataloger, enabled: false config :explorer, Explorer.Tracer, disabled?: false diff --git a/apps/explorer/lib/explorer/application.ex b/apps/explorer/lib/explorer/application.ex index 9645572b62..53d8c03ea3 100644 --- a/apps/explorer/lib/explorer/application.ex +++ b/apps/explorer/lib/explorer/application.ex @@ -68,6 +68,7 @@ defmodule Explorer.Application do configure(Explorer.KnownTokens), configure(Explorer.Market.History.Cataloger), configure(Explorer.Counters.AddressesWithBalanceCounter), + configure(Explorer.Counters.AddressesCounter), configure(Explorer.Counters.AverageBlockTime), configure(Explorer.Validator.MetadataProcessor), configure(Explorer.Staking.EpochCounter) diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index 36cfae424e..f89d933d6d 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -58,7 +58,7 @@ defmodule Explorer.Chain do } alias Explorer.Chain.Import.Runner - alias Explorer.Counters.AddressesWithBalanceCounter + alias Explorer.Counters.{AddressesCounter, AddressesWithBalanceCounter} alias Explorer.Market.MarketHistoryCache alias Explorer.{PagingOptions, Repo} @@ -118,6 +118,14 @@ defmodule Explorer.Chain do AddressesWithBalanceCounter.fetch() end + @doc """ + Gets from the cache the count of all `t:Explorer.Chain.Address.t/0`'s + """ + @spec count_addresses_from_cache :: non_neg_integer() + def count_addresses_from_cache do + AddressesCounter.fetch() + end + @doc """ Counts the number of addresses with fetched coin balance > 0. @@ -131,6 +139,19 @@ defmodule Explorer.Chain do ) end + @doc """ + Counts the number of all addresses. + + This function should be used with caution. In larger databases, it may take a + while to have the return back. + """ + def count_addresses do + Repo.one( + Address.count(), + timeout: :infinity + ) + end + @doc """ `t:Explorer.Chain.InternalTransaction/0`s from the address with the given `hash`. diff --git a/apps/explorer/lib/explorer/chain/address.ex b/apps/explorer/lib/explorer/chain/address.ex index 96d4dc7ab4..76ea5e85dd 100644 --- a/apps/explorer/lib/explorer/chain/address.ex +++ b/apps/explorer/lib/explorer/chain/address.ex @@ -237,6 +237,16 @@ defmodule Explorer.Chain.Address do ) end + @doc """ + Counts all the addresses. + """ + def count do + from( + a in Address, + select: fragment("COUNT(*)") + ) + end + defimpl String.Chars do @doc """ Uses `hash` as string representation, formatting it according to the eip-55 specification diff --git a/apps/explorer/lib/explorer/counters/addresses_counter.ex b/apps/explorer/lib/explorer/counters/addresses_counter.ex new file mode 100644 index 0000000000..5febd8fbb9 --- /dev/null +++ b/apps/explorer/lib/explorer/counters/addresses_counter.ex @@ -0,0 +1,125 @@ +defmodule Explorer.Counters.AddressesCounter do + @moduledoc """ + Caches the number of all addresses. + + It loads the count asynchronously and in a time interval of 30 minutes. + """ + + use GenServer + + alias Explorer.Chain + + @table :addresses_counter + + @cache_key "addresses" + + def table_name do + @table + end + + def cache_key do + @cache_key + end + + # It is undesirable to automatically start the consolidation in all environments. + # Consider the test environment: if the consolidation initiates but does not + # finish before a test ends, that test will fail. This way, hundreds of + # tests were failing before disabling the consolidation and the scheduler in + # the test env. + config = Application.get_env(:explorer, Explorer.Counters.AddressesCounter) + @enable_consolidation Keyword.get(config, :enable_consolidation) + + @update_interval_in_seconds Keyword.get(config, :update_interval_in_seconds) + + @doc """ + Starts a process to periodically update the counter of the token holders. + """ + @spec start_link(term()) :: GenServer.on_start() + def start_link(_) do + GenServer.start_link(__MODULE__, :ok, name: __MODULE__) + end + + @impl true + def init(_args) do + create_table() + + {:ok, %{consolidate?: enable_consolidation?()}, {:continue, :ok}} + end + + def create_table do + opts = [ + :set, + :named_table, + :public, + read_concurrency: true + ] + + :ets.new(table_name(), opts) + end + + defp schedule_next_consolidation do + Process.send_after(self(), :consolidate, :timer.seconds(@update_interval_in_seconds)) + end + + @doc """ + Inserts new items into the `:ets` table. + """ + def insert_counter({key, info}) do + :ets.insert(table_name(), {key, info}) + end + + @impl true + def handle_continue(:ok, %{consolidate?: true} = state) do + consolidate() + schedule_next_consolidation() + + {:noreply, state} + end + + @impl true + def handle_continue(:ok, state) do + {:noreply, state} + end + + @impl true + def handle_info(:consolidate, state) do + consolidate() + + schedule_next_consolidation() + + {:noreply, state} + end + + @doc """ + Fetches the info for a specific item from the `:ets` table. + """ + def fetch do + do_fetch(:ets.lookup(table_name(), cache_key())) + end + + defp do_fetch([{_, result}]), do: result + defp do_fetch([]), do: 0 + + @doc """ + Consolidates the info by populating the `:ets` table with the current database information. + """ + def consolidate do + counter = Chain.count_addresses() + + insert_counter({cache_key(), counter}) + end + + @doc """ + Returns a boolean that indicates whether consolidation is enabled + + In order to choose whether or not to enable the scheduler and the initial + consolidation, change the following Explorer config: + + `config :explorer, Explorer.Counters.AddressesCounter, enable_consolidation: true` + + to: + + `config :explorer, Explorer.Counters.AddressesCounter, enable_consolidation: false` + """ + def enable_consolidation?, do: @enable_consolidation +end diff --git a/apps/explorer/test/explorer/chain_test.exs b/apps/explorer/test/explorer/chain_test.exs index 3ac60b6963..872a3bc5c0 100644 --- a/apps/explorer/test/explorer/chain_test.exs +++ b/apps/explorer/test/explorer/chain_test.exs @@ -27,6 +27,7 @@ defmodule Explorer.ChainTest do alias Explorer.Chain.Supply.ProofOfAuthority alias Explorer.Counters.AddressesWithBalanceCounter + alias Explorer.Counters.AddressesCounter doctest Explorer.Chain @@ -50,6 +51,22 @@ defmodule Explorer.ChainTest do end end + describe "count_addresses_from_cache/0" do + test "returns the number of all addresses" do + insert(:address, fetched_coin_balance: 0) + insert(:address, fetched_coin_balance: 1) + insert(:address, fetched_coin_balance: 2) + + start_supervised!(AddressesCounter) + AddressesCounter.consolidate() + + addresses_with_balance = Chain.count_addresses_from_cache() + + assert is_integer(addresses_with_balance) + assert addresses_with_balance == 3 + end + end + describe "last_db_block_status/0" do test "return no_blocks errors if db is empty" do assert {:error, :no_blocks} = Chain.last_db_block_status() diff --git a/apps/explorer/test/explorer/counters/addresses_counter_test.exs b/apps/explorer/test/explorer/counters/addresses_counter_test.exs new file mode 100644 index 0000000000..b187fd8c0e --- /dev/null +++ b/apps/explorer/test/explorer/counters/addresses_counter_test.exs @@ -0,0 +1,16 @@ +defmodule Explorer.Counters.AddressesCounterTest do + use Explorer.DataCase + + alias Explorer.Counters.AddressesCounter + + test "populates the cache with the number of all addresses" do + insert(:address, fetched_coin_balance: 0) + insert(:address, fetched_coin_balance: 1) + insert(:address, fetched_coin_balance: 2) + + start_supervised!(AddressesCounter) + AddressesCounter.consolidate() + + assert AddressesCounter.fetch() == 3 + end +end From a5062595378ebb6fc7762d587a7c2c83e00a9010 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 24 Sep 2019 12:59:16 +0300 Subject: [PATCH 63/67] fix library verification bytecode of a library contains address of a deployed library --- .../lib/explorer/smart_contract/verifier.ex | 14 +++++++++- .../explorer/smart_contract/verifier_test.exs | 28 +++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/apps/explorer/lib/explorer/smart_contract/verifier.ex b/apps/explorer/lib/explorer/smart_contract/verifier.ex index 4c2daa680c..4303cf5b6b 100644 --- a/apps/explorer/lib/explorer/smart_contract/verifier.ex +++ b/apps/explorer/lib/explorer/smart_contract/verifier.ex @@ -69,7 +69,8 @@ defmodule Explorer.SmartContract.Verifier do blockchain_bytecode_without_whisper = extract_bytecode(blockchain_bytecode) cond do - generated_bytecode != blockchain_bytecode_without_whisper -> + generated_bytecode != blockchain_bytecode_without_whisper && + !try_library_verification(generated_bytecode, blockchain_bytecode_without_whisper) -> {:error, :generated_bytecode} has_constructor_with_params?(abi) && @@ -81,6 +82,17 @@ defmodule Explorer.SmartContract.Verifier do end end + defp try_library_verification( + "730000000000000000000000000000000000000000" <> bytecode, + <<_address::binary-size(42)>> <> bytecode + ) do + true + end + + defp try_library_verification(_, _) do + false + end + @doc """ In order to discover the bytecode we need to remove the `swarm source` from the hash. diff --git a/apps/explorer/test/explorer/smart_contract/verifier_test.exs b/apps/explorer/test/explorer/smart_contract/verifier_test.exs index fba0622964..088820beea 100644 --- a/apps/explorer/test/explorer/smart_contract/verifier_test.exs +++ b/apps/explorer/test/explorer/smart_contract/verifier_test.exs @@ -144,6 +144,34 @@ defmodule Explorer.SmartContract.VerifierTest do assert abi != nil end + test "verifies a library" do + bytecode = + "0x7349f540c22cba15c47a08c235e20081474201a742301460806040526004361060335760003560e01c8063c2985578146038575b600080fd5b603e60b0565b6040805160208082528351818301528351919283929083019185019080838360005b8381101560765781810151838201526020016060565b50505050905090810190601f16801560a25780820380516001836020036101000a031916815260200191505b509250505060405180910390f35b604080518082019091526003815262666f6f60e81b60208201529056fea265627a7a72315820174b282a3ef3b9778d79fbc2e4c36bc939c54dfaaaa51d3122ee6e648093844c64736f6c634300050b0032" + + contract_address = insert(:contract_address, contract_code: bytecode) + + code = """ + pragma solidity 0.5.11; + + library Foo { + function foo() external pure returns (string memory) { + return "foo"; + } + } + """ + + params = %{ + "contract_source_code" => code, + "compiler_version" => "v0.5.11+commit.c082d0b4", + "evm_version" => "default", + "name" => "Foo", + "optimization" => true + } + + assert {:ok, %{abi: abi}} = Verifier.evaluate_authenticity(contract_address.hash, params) + assert abi != nil + end + test "verifies smart contract compiled with Solidity 0.5.9 (includes new metadata in bytecode) with constructor args" do path = File.cwd!() <> "/test/support/fixture/smart_contract/solidity_0.5.9_smart_contract.sol" contract = File.read!(path) From 6b442635ebe8123a8c2559737cfe72742aab0526 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 24 Sep 2019 13:02:24 +0300 Subject: [PATCH 64/67] add CHANGELOG entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c805d8f45..4a53de6a31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ - [#2663](https://github.com/poanetwork/blockscout/pull/2663) - Fetch address counters in parallel ### Fixes +- [#2731](https://github.com/poanetwork/blockscout/pull/2731) - fix library verification - [#2718](https://github.com/poanetwork/blockscout/pull/2718) - Include all addresses taking part in transactions in wallets' addresses counter - [#2709](https://github.com/poanetwork/blockscout/pull/2709) - Fix stuck label and value for uncle block height - [#2707](https://github.com/poanetwork/blockscout/pull/2707) - fix for dashboard banner chart legend items From ec20050a662e8a37c5506944b3e021fa2198e83f Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 24 Sep 2019 14:14:21 +0300 Subject: [PATCH 65/67] add comment --- apps/explorer/lib/explorer/smart_contract/verifier.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/explorer/lib/explorer/smart_contract/verifier.ex b/apps/explorer/lib/explorer/smart_contract/verifier.ex index 4303cf5b6b..ed1b6532c5 100644 --- a/apps/explorer/lib/explorer/smart_contract/verifier.ex +++ b/apps/explorer/lib/explorer/smart_contract/verifier.ex @@ -82,6 +82,7 @@ defmodule Explorer.SmartContract.Verifier do end end + # 730000000000000000000000000000000000000000 - default library address that returned by the compiler defp try_library_verification( "730000000000000000000000000000000000000000" <> bytecode, <<_address::binary-size(42)>> <> bytecode From 156c6ca4551036ff42fc9c8df8cbea371f523a65 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Wed, 25 Sep 2019 13:36:50 +0300 Subject: [PATCH 66/67] do not update cache if no blocks were inserted --- apps/indexer/lib/indexer/block/fetcher.ex | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/indexer/lib/indexer/block/fetcher.ex b/apps/indexer/lib/indexer/block/fetcher.ex index 1b95d84086..85b47cba6f 100644 --- a/apps/indexer/lib/indexer/block/fetcher.ex +++ b/apps/indexer/lib/indexer/block/fetcher.ex @@ -184,6 +184,8 @@ defmodule Indexer.Block.Fetcher do end end + defp update_block_cache([]), do: :ok + defp update_block_cache(blocks) when is_list(blocks) do {min_block, max_block} = Enum.min_max_by(blocks, & &1.number) From 3fa83faf2caeaaada3445d9b8257c80753ab3144 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Wed, 25 Sep 2019 13:39:09 +0300 Subject: [PATCH 67/67] add CHANGELOG entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6791e814b7..da3cf2b091 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ - [#2663](https://github.com/poanetwork/blockscout/pull/2663) - Fetch address counters in parallel ### Fixes +- [#2736](https://github.com/poanetwork/blockscout/pull/2736) - do not update cache if no blocks were inserted - [#2718](https://github.com/poanetwork/blockscout/pull/2718) - Include all addresses taking part in transactions in wallets' addresses counter - [#2709](https://github.com/poanetwork/blockscout/pull/2709) - Fix stuck label and value for uncle block height - [#2707](https://github.com/poanetwork/blockscout/pull/2707) - fix for dashboard banner chart legend items