From 3c90107c92e63071903a9a141a50f713a54b1203 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Mon, 22 Apr 2019 15:46:40 +0300 Subject: [PATCH 01/12] use coinmarketcap for total_supply by default --- apps/explorer/lib/explorer/chain.ex | 2 +- .../explorer/chain/supply/coin_market_cap.ex | 22 +++++++++++++++++++ .../exchange_rates/source/coin_gecko.ex | 1 + .../exchange_rates/source/coin_market_cap.ex | 1 + .../source/transaction_and_log.ex | 1 + .../lib/explorer/exchange_rates/token.ex | 15 +++++++++---- apps/explorer/test/explorer/chain_test.exs | 2 ++ .../test/support/fakes/one_coin_source.ex | 1 + 8 files changed, 40 insertions(+), 5 deletions(-) create mode 100644 apps/explorer/lib/explorer/chain/supply/coin_market_cap.ex diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index 7ccf1d0606..590bca23c5 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -2374,7 +2374,7 @@ defmodule Explorer.Chain do end defp supply_module do - Application.get_env(:explorer, :supply, Explorer.Chain.Supply.ProofOfAuthority) + Application.get_env(:explorer, :supply, Explorer.Chain.Supply.CoinMarketCap) end @doc """ diff --git a/apps/explorer/lib/explorer/chain/supply/coin_market_cap.ex b/apps/explorer/lib/explorer/chain/supply/coin_market_cap.ex new file mode 100644 index 0000000000..ebaadb3c47 --- /dev/null +++ b/apps/explorer/lib/explorer/chain/supply/coin_market_cap.ex @@ -0,0 +1,22 @@ +defmodule Explorer.Chain.Supply.CoinMarketCap do + @moduledoc """ + Defines the supply API for calculating supply for coins from coinmarketcap. + """ + + use Explorer.Chain.Supply + + alias Explorer.ExchangeRates.Token + alias Explorer.Market + + def circulating do + exchange_rate().available_supply + end + + def total do + exchange_rate().total_supply + end + + def exchange_rate do + Market.get_exchange_rate(Explorer.coin()) || Token.null() + end +end diff --git a/apps/explorer/lib/explorer/exchange_rates/source/coin_gecko.ex b/apps/explorer/lib/explorer/exchange_rates/source/coin_gecko.ex index 4ad4c98292..4e59537bf6 100644 --- a/apps/explorer/lib/explorer/exchange_rates/source/coin_gecko.ex +++ b/apps/explorer/lib/explorer/exchange_rates/source/coin_gecko.ex @@ -26,6 +26,7 @@ defmodule Explorer.ExchangeRates.Source.CoinGecko do %Token{ available_supply: to_decimal(item["total_supply"]), + total_supply: to_decimal(item["total_supply"]), btc_value: btc_value, id: id, last_updated: last_updated, diff --git a/apps/explorer/lib/explorer/exchange_rates/source/coin_market_cap.ex b/apps/explorer/lib/explorer/exchange_rates/source/coin_market_cap.ex index 4030eed06f..0e8521f4ee 100644 --- a/apps/explorer/lib/explorer/exchange_rates/source/coin_market_cap.ex +++ b/apps/explorer/lib/explorer/exchange_rates/source/coin_market_cap.ex @@ -17,6 +17,7 @@ defmodule Explorer.ExchangeRates.Source.CoinMarketCap do %Token{ available_supply: to_decimal(item["available_supply"]), + total_supply: to_decimal(item["total_supply"]), btc_value: to_decimal(item["price_btc"]), id: item["id"], last_updated: last_updated, diff --git a/apps/explorer/lib/explorer/exchange_rates/source/transaction_and_log.ex b/apps/explorer/lib/explorer/exchange_rates/source/transaction_and_log.ex index a3dfafecaa..2b8699accc 100644 --- a/apps/explorer/lib/explorer/exchange_rates/source/transaction_and_log.ex +++ b/apps/explorer/lib/explorer/exchange_rates/source/transaction_and_log.ex @@ -26,6 +26,7 @@ defmodule Explorer.ExchangeRates.Source.TransactionAndLog do defp build_struct(original_token) do %Token{ available_supply: to_decimal(Chain.circulating_supply()), + total_supply: 0, btc_value: original_token.btc_value, id: original_token.id, last_updated: original_token.last_updated, diff --git a/apps/explorer/lib/explorer/exchange_rates/token.ex b/apps/explorer/lib/explorer/exchange_rates/token.ex index 3df8cbd56b..6521181c12 100644 --- a/apps/explorer/lib/explorer/exchange_rates/token.ex +++ b/apps/explorer/lib/explorer/exchange_rates/token.ex @@ -7,6 +7,7 @@ defmodule Explorer.ExchangeRates.Token do Represents an exchange rate for a given token. * `:available_supply` - Available supply of a token + * `:total_supply` - Max Supply * `:btc_value` - The Bitcoin value of the currency * `:id` - ID of a currency * `:last_updated` - Timestamp of when the value was last updated @@ -18,6 +19,7 @@ defmodule Explorer.ExchangeRates.Token do """ @type t :: %__MODULE__{ available_supply: Decimal.t(), + total_supply: Decimal.t(), btc_value: Decimal.t(), id: String.t(), last_updated: DateTime.t(), @@ -28,8 +30,8 @@ defmodule Explorer.ExchangeRates.Token do volume_24h_usd: Decimal.t() } - @enforce_keys ~w(available_supply btc_value id last_updated market_cap_usd name symbol usd_value volume_24h_usd)a - defstruct ~w(available_supply btc_value id last_updated market_cap_usd name symbol usd_value volume_24h_usd)a + @enforce_keys ~w(available_supply total_supply btc_value id last_updated market_cap_usd name symbol usd_value volume_24h_usd)a + defstruct ~w(available_supply total_supply btc_value id last_updated market_cap_usd name symbol usd_value volume_24h_usd)a def null, do: %__MODULE__{ @@ -37,6 +39,7 @@ defmodule Explorer.ExchangeRates.Token do id: nil, name: nil, available_supply: nil, + total_supply: nil, usd_value: nil, volume_24h_usd: nil, market_cap_usd: nil, @@ -51,6 +54,7 @@ defmodule Explorer.ExchangeRates.Token do id: id, name: name, available_supply: available_supply, + total_supply: total_supply, usd_value: usd_value, volume_24h_usd: volume_24h_usd, market_cap_usd: market_cap_usd, @@ -58,17 +62,20 @@ defmodule Explorer.ExchangeRates.Token do last_updated: last_updated }) do # symbol is first because it is the key used for lookup in `Explorer.ExchangeRates`'s ETS table - {symbol, id, name, available_supply, usd_value, volume_24h_usd, market_cap_usd, btc_value, last_updated} + {symbol, id, name, available_supply, total_supply, usd_value, volume_24h_usd, market_cap_usd, btc_value, + last_updated} end def from_tuple( - {symbol, id, name, available_supply, usd_value, volume_24h_usd, market_cap_usd, btc_value, last_updated} + {symbol, id, name, available_supply, total_supply, usd_value, volume_24h_usd, market_cap_usd, btc_value, + last_updated} ) do %__MODULE__{ symbol: symbol, id: id, name: name, available_supply: available_supply, + total_supply: total_supply, usd_value: usd_value, volume_24h_usd: volume_24h_usd, market_cap_usd: market_cap_usd, diff --git a/apps/explorer/test/explorer/chain_test.exs b/apps/explorer/test/explorer/chain_test.exs index 8b04eb922e..0087a607e7 100644 --- a/apps/explorer/test/explorer/chain_test.exs +++ b/apps/explorer/test/explorer/chain_test.exs @@ -3273,6 +3273,7 @@ defmodule Explorer.ChainTest do end test "total_supply/0" do + Application.put_env(:explorer, :supply, Explorer.Chain.Supply.ProofOfAuthority) height = 2_000_000 insert(:block, number: height) expected = ProofOfAuthority.initial_supply() + height @@ -3281,6 +3282,7 @@ defmodule Explorer.ChainTest do end test "circulating_supply/0" do + Application.put_env(:explorer, :supply, Explorer.Chain.Supply.ProofOfAuthority) assert Chain.circulating_supply() == ProofOfAuthority.circulating() end diff --git a/apps/explorer/test/support/fakes/one_coin_source.ex b/apps/explorer/test/support/fakes/one_coin_source.ex index 73bb104866..72ba399bf9 100644 --- a/apps/explorer/test/support/fakes/one_coin_source.ex +++ b/apps/explorer/test/support/fakes/one_coin_source.ex @@ -10,6 +10,7 @@ defmodule Explorer.ExchangeRates.Source.OneCoinSource do def format_data(_) do pseudo_token = %Token{ available_supply: Decimal.new(10_000_000), + total_supply: Decimal.new(10_000_000_000), btc_value: Decimal.new(1), id: "", last_updated: Timex.now(), From 28ade9ed74da82a5641187d224e57d62ec1a542f Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Mon, 22 Apr 2019 15:50:53 +0300 Subject: [PATCH 02/12] add CHANGELOG entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 03a2380828..5db2f313f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ - [#1790](https://github.com/poanetwork/blockscout/pull/1790) - fix constructor arguments verification - [#1793](https://github.com/poanetwork/blockscout/pull/1793) - fix top nav autocomplete - [#1795](https://github.com/poanetwork/blockscout/pull/1795) - fix line numbers for decompiled contracts + - [#1803](https://github.com/poanetwork/blockscout/pull/1803) - use coinmarketcap for total_supply by default ### Chore From 9c99be8db16e16db4ed585a7b07768198ff32d0b Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Mon, 22 Apr 2019 16:00:14 +0300 Subject: [PATCH 03/12] fix total_supply in test --- .../test/explorer/exchange_rates/source/coin_market_cap_test.exs | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/explorer/test/explorer/exchange_rates/source/coin_market_cap_test.exs b/apps/explorer/test/explorer/exchange_rates/source/coin_market_cap_test.exs index 888cb5875c..e0f1693212 100644 --- a/apps/explorer/test/explorer/exchange_rates/source/coin_market_cap_test.exs +++ b/apps/explorer/test/explorer/exchange_rates/source/coin_market_cap_test.exs @@ -33,6 +33,7 @@ defmodule Explorer.ExchangeRates.Source.CoinMarketCapTest do expected = [ %Token{ available_supply: Decimal.new("203981804.0"), + total_supply: Decimal.new("254473964.0"), btc_value: Decimal.new("0.00007032"), id: "poa-network", last_updated: expected_date, From 3bff1672512d42834c2a534a85ef2446e7ab035e Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Mon, 22 Apr 2019 16:11:52 +0300 Subject: [PATCH 04/12] fix remaining tests --- .../test/explorer/exchange_rates/exchange_rates_test.exs | 1 + .../test/explorer/exchange_rates/source/coin_gecko_test.exs | 1 + 2 files changed, 2 insertions(+) diff --git a/apps/explorer/test/explorer/exchange_rates/exchange_rates_test.exs b/apps/explorer/test/explorer/exchange_rates/exchange_rates_test.exs index 254c7469ba..1d44a7adce 100644 --- a/apps/explorer/test/explorer/exchange_rates/exchange_rates_test.exs +++ b/apps/explorer/test/explorer/exchange_rates/exchange_rates_test.exs @@ -68,6 +68,7 @@ defmodule Explorer.ExchangeRatesTest do test "with successful fetch" do expected_token = %Token{ available_supply: Decimal.new("1000000.0"), + total_supply: Decimal.new("1000000.0"), btc_value: Decimal.new("1.000"), id: "test_id", last_updated: DateTime.utc_now(), diff --git a/apps/explorer/test/explorer/exchange_rates/source/coin_gecko_test.exs b/apps/explorer/test/explorer/exchange_rates/source/coin_gecko_test.exs index 5317426ddd..3c2a9b6feb 100644 --- a/apps/explorer/test/explorer/exchange_rates/source/coin_gecko_test.exs +++ b/apps/explorer/test/explorer/exchange_rates/source/coin_gecko_test.exs @@ -64,6 +64,7 @@ defmodule Explorer.ExchangeRates.Source.CoinGeckoTest do expected = [ %Token{ available_supply: Decimal.new("252193195"), + total_supply: Decimal.new("252193195"), btc_value: Decimal.new("0.00001753101509231471092879666458"), id: "poa-network", last_updated: expected_date, From 2562640bf3e30f16dc1d9b45a8fe06891aa632a8 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Mon, 22 Apr 2019 16:28:23 +0300 Subject: [PATCH 05/12] fix remaining tests --- .../block_scout_web/channels/exchange_rate_channel_test.exs | 1 + .../controllers/api/rpc/stats_controller_test.exs | 1 + .../test/block_scout_web/views/address_view_test.exs | 2 ++ 3 files changed, 4 insertions(+) diff --git a/apps/block_scout_web/test/block_scout_web/channels/exchange_rate_channel_test.exs b/apps/block_scout_web/test/block_scout_web/channels/exchange_rate_channel_test.exs index 7c495e6bfc..acdf441750 100644 --- a/apps/block_scout_web/test/block_scout_web/channels/exchange_rate_channel_test.exs +++ b/apps/block_scout_web/test/block_scout_web/channels/exchange_rate_channel_test.exs @@ -21,6 +21,7 @@ defmodule BlockScoutWeb.ExchangeRateChannelTest do token = %Token{ available_supply: Decimal.new("1000000.0"), + total_supply: Decimal.new("1000000.0"), btc_value: Decimal.new("1.000"), id: "test", last_updated: DateTime.utc_now(), diff --git a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/stats_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/stats_controller_test.exs index fe8311daf1..7c47ed622e 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/stats_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/stats_controller_test.exs @@ -122,6 +122,7 @@ defmodule BlockScoutWeb.API.RPC.StatsControllerTest do eth = %Token{ available_supply: Decimal.new("1000000.0"), + total_supply: Decimal.new("1000000.0"), btc_value: Decimal.new("1.000"), id: "test", last_updated: DateTime.utc_now(), diff --git a/apps/block_scout_web/test/block_scout_web/views/address_view_test.exs b/apps/block_scout_web/test/block_scout_web/views/address_view_test.exs index 6219017c1a..0809850163 100644 --- a/apps/block_scout_web/test/block_scout_web/views/address_view_test.exs +++ b/apps/block_scout_web/test/block_scout_web/views/address_view_test.exs @@ -134,7 +134,9 @@ defmodule BlockScoutWeb.AddressViewTest do end test "balance_percentage/1" do + Application.put_env(:explorer, :supply, Explorer.Chain.Supply.ProofOfAuthority) address = insert(:address, fetched_coin_balance: 2_524_608_000_000_000_000_000_000) + assert "1.0000% Market Cap" = AddressView.balance_percentage(address) end From d36a7d7ca8b11db03bb7e011b18f25eb9703a47b Mon Sep 17 00:00:00 2001 From: Paul Tsupikoff Date: Mon, 22 Apr 2019 17:13:57 +0300 Subject: [PATCH 06/12] Disable lazy logging check in Credo (#1800) Logger macros don't evaluate their parameters since [Elixir 1.7.0](https://elixir-lang.org/blog/2018/07/25/elixir-v1-7-0-released/). Creating a lambda for every log output is no longer needed. --- .credo.exs | 4 +++- CHANGELOG.md | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.credo.exs b/.credo.exs index 4dda48adf4..ab3359abb1 100644 --- a/.credo.exs +++ b/.credo.exs @@ -67,6 +67,9 @@ {Credo.Check.Readability.TrailingBlankLine, false}, {Credo.Check.Readability.TrailingWhiteSpace, false}, + # outdated by lazy Logger in Elixir 1.7. See https://elixir-lang.org/blog/2018/07/25/elixir-v1-7-0-released/ + {Credo.Check.Warning.LazyLogging, false}, + # not handled by formatter {Credo.Check.Consistency.ExceptionNames}, {Credo.Check.Consistency.ParameterPatternMatching}, @@ -118,7 +121,6 @@ {Credo.Check.Warning.ExpensiveEmptyEnumCheck}, {Credo.Check.Warning.IExPry}, {Credo.Check.Warning.IoInspect}, - {Credo.Check.Warning.LazyLogging}, {Credo.Check.Warning.OperationOnSameValues}, {Credo.Check.Warning.OperationWithConstantResult}, {Credo.Check.Warning.UnusedEnumOperation}, diff --git a/CHANGELOG.md b/CHANGELOG.md index aab08669f8..b9f18ede21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ - [#1753](https://github.com/poanetwork/blockscout/pull/1753) - Add a check mark to decompiled contract tab - [#1744](https://github.com/poanetwork/blockscout/pull/1744) - remove `0x0..0` from tests - [#1763](https://github.com/poanetwork/blockscout/pull/1763) - Describe indexer structure and list existing fetchers + - [#1800](https://github.com/poanetwork/blockscout/pull/1800) - Disable lazy logging check in Credo ## 1.3.9-beta From 8e6ee4ceb1e0a51996bbf072810a223e746f2d35 Mon Sep 17 00:00:00 2001 From: goodsoft Date: Sun, 21 Apr 2019 19:24:44 +0300 Subject: [PATCH 07/12] Use eth_getUncleByBlockHashAndIndex for uncle block fetching Parity returns `null` for many uncle blocks, if they are fetched using `eth_getBlockByHash`. This is most probably caused by the fact that Parity only keeps full block data for recent non-consensus blocks. This causes uncle_block fetcher to loop endlessly trying to fetch "missing" blocks, hogging both app server and Parity resources. Instead, we use `eth_getUncleByBlockHashAndIndex` method, which works for all uncle blocks. As we didn't previously store index of an uncle block within a nephew block, a new field and a temporary fixup fetcher is added to get the index from nephew blocks. --- CHANGELOG.md | 1 + apps/ethereum_jsonrpc/lib/ethereum_jsonrpc.ex | 14 ++ .../lib/ethereum_jsonrpc/block.ex | 7 +- .../lib/ethereum_jsonrpc/block/by_nephew.ex | 15 ++ .../lib/ethereum_jsonrpc/blocks.ex | 3 +- .../lib/ethereum_jsonrpc/uncle.ex | 17 +- .../lib/ethereum_jsonrpc/uncles.ex | 6 +- apps/explorer/lib/explorer/chain.ex | 15 +- .../chain/block/second_degree_relation.ex | 10 +- .../runner/block/second_degree_relations.ex | 25 ++- .../20190421143300_add_index_to_bsdr.exs | 10 ++ .../block/second_degree_relation_test.exs | 16 +- .../test/explorer/chain/import_test.exs | 2 +- apps/explorer/test/explorer/chain_test.exs | 13 +- apps/explorer/test/support/factory.ex | 3 +- apps/indexer/README.md | 1 + apps/indexer/lib/indexer/block/fetcher.ex | 4 +- .../lib/indexer/fetcher/uncle_block.ex | 55 +++--- apps/indexer/lib/indexer/supervisor.ex | 7 +- .../indexer/temporary/uncles_without_index.ex | 164 ++++++++++++++++++ .../indexer/block/catchup/fetcher_test.exs | 11 +- .../test/indexer/fetcher/uncle_block_test.exs | 20 ++- 22 files changed, 340 insertions(+), 79 deletions(-) create mode 100644 apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/block/by_nephew.ex create mode 100644 apps/explorer/priv/repo/migrations/20190421143300_add_index_to_bsdr.exs create mode 100644 apps/indexer/lib/indexer/temporary/uncles_without_index.ex diff --git a/CHANGELOG.md b/CHANGELOG.md index b9f18ede21..9ad6ec7098 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ - [#1793](https://github.com/poanetwork/blockscout/pull/1793) - fix top nav autocomplete - [#1795](https://github.com/poanetwork/blockscout/pull/1795) - fix line numbers for decompiled contracts - [#1802](https://github.com/poanetwork/blockscout/pull/1802) - make coinmarketcap's number of pages configurable + - [#1799](https://github.com/poanetwork/blockscout/pull/1799) - Use eth_getUncleByBlockHashAndIndex for uncle block fetching ### Chore diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc.ex index 33d3abd31b..24fc593d85 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc.ex @@ -52,6 +52,11 @@ defmodule EthereumJSONRPC do """ @type block_number :: non_neg_integer() + @typedoc """ + Reference to an uncle block by nephew block's `hash` and `index` in it. + """ + @type nephew_index :: %{required(:nephew_hash) => String.t(), required(:index) => non_neg_integer()} + @typedoc """ Binary data encoded as a single hexadecimal number in a `String.t` """ @@ -235,6 +240,15 @@ defmodule EthereumJSONRPC do |> fetch_blocks_by_params(&Block.ByNumber.request/1, json_rpc_named_arguments) end + @doc """ + Fetches uncle blocks by nephew hashes and indices. + """ + @spec fetch_uncle_blocks([nephew_index()], json_rpc_named_arguments) :: {:ok, Blocks.t()} | {:error, reason :: term} + def fetch_uncle_blocks(blocks, json_rpc_named_arguments) do + blocks + |> fetch_blocks_by_params(&Block.ByNephew.request/1, json_rpc_named_arguments) + end + @doc """ Fetches block number by `t:tag/0`. diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/block.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/block.ex index 3aa6eed029..b2f3a87fcd 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/block.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/block.ex @@ -356,14 +356,17 @@ defmodule EthereumJSONRPC.Block do [ %{ "hash" => "0xe670ec64341771606e55d6b4ca35a1a6b75ee3d5145a99d05921026d15273311", - "nephewHash" => "0xe52d77084cab13a4e724162bcd8c6028e5ecfaa04d091ee476e96b9958ed6b47" + "nephewHash" => "0xe52d77084cab13a4e724162bcd8c6028e5ecfaa04d091ee476e96b9958ed6b47", + "index" => 0 } ] """ @spec elixir_to_uncles(elixir) :: Uncles.elixir() def elixir_to_uncles(%{"hash" => nephew_hash, "uncles" => uncles}) do - Enum.map(uncles, &%{"hash" => &1, "nephewHash" => nephew_hash}) + uncles + |> Enum.with_index() + |> Enum.map(fn {uncle_hash, index} -> %{"hash" => uncle_hash, "nephewHash" => nephew_hash, "index" => index} end) end @doc """ diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/block/by_nephew.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/block/by_nephew.ex new file mode 100644 index 0000000000..56e8393343 --- /dev/null +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/block/by_nephew.ex @@ -0,0 +1,15 @@ +defmodule EthereumJSONRPC.Block.ByNephew do + @moduledoc """ + Block format as returned by [`eth_getUncleByBlockHashAndIndex`](https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_getUncleByBlockHashAndIndex) + """ + + import EthereumJSONRPC, only: [integer_to_quantity: 1] + + def request(%{id: id, nephew_hash: nephew_hash, index: index}) do + EthereumJSONRPC.request(%{ + id: id, + method: "eth_getUncleByBlockHashAndIndex", + params: [nephew_hash, integer_to_quantity(index)] + }) + end +end diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/blocks.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/blocks.ex index 22bb74faf6..dc1740a4aa 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/blocks.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/blocks.ex @@ -260,7 +260,8 @@ defmodule EthereumJSONRPC.Blocks do [ %{ "hash" => "0xe670ec64341771606e55d6b4ca35a1a6b75ee3d5145a99d05921026d15273311", - "nephewHash" => "0xe52d77084cab13a4e724162bcd8c6028e5ecfaa04d091ee476e96b9958ed6b47" + "nephewHash" => "0xe52d77084cab13a4e724162bcd8c6028e5ecfaa04d091ee476e96b9958ed6b47", + "index" => 0 } ] diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/uncle.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/uncle.ex index 871f0daa7d..fb0a6a397e 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/uncle.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/uncle.ex @@ -6,15 +6,16 @@ defmodule EthereumJSONRPC.Uncle do chain. """ - @type elixir :: %{String.t() => EthereumJSONRPC.hash()} + @type elixir :: %{String.t() => EthereumJSONRPC.hash() | non_neg_integer()} @typedoc """ * `"hash"` - the hash of the uncle block. * `"nephewHash"` - the hash of the nephew block that included `"hash` as an uncle. + * `"index"` - the index of the uncle block within the nephew block. """ @type t :: %{String.t() => EthereumJSONRPC.hash()} - @type params :: %{nephew_hash: EthereumJSONRPC.hash(), uncle_hash: EthereumJSONRPC.hash()} + @type params :: %{nephew_hash: EthereumJSONRPC.hash(), uncle_hash: EthereumJSONRPC.hash(), index: non_neg_integer()} @doc """ Converts each entry in `t:elixir/0` to `t:params/0` used in `Explorer.Chain.Uncle.changeset/2`. @@ -22,18 +23,20 @@ defmodule EthereumJSONRPC.Uncle do iex> EthereumJSONRPC.Uncle.elixir_to_params( ...> %{ ...> "hash" => "0xe670ec64341771606e55d6b4ca35a1a6b75ee3d5145a99d05921026d15273311", - ...> "nephewHash" => "0xe52d77084cab13a4e724162bcd8c6028e5ecfaa04d091ee476e96b9958ed6b47" + ...> "nephewHash" => "0xe52d77084cab13a4e724162bcd8c6028e5ecfaa04d091ee476e96b9958ed6b47", + ...> "index" => 0 ...> } ...> ) %{ nephew_hash: "0xe52d77084cab13a4e724162bcd8c6028e5ecfaa04d091ee476e96b9958ed6b47", - uncle_hash: "0xe670ec64341771606e55d6b4ca35a1a6b75ee3d5145a99d05921026d15273311" + uncle_hash: "0xe670ec64341771606e55d6b4ca35a1a6b75ee3d5145a99d05921026d15273311", + index: 0 } """ @spec elixir_to_params(elixir) :: params - def elixir_to_params(%{"hash" => uncle_hash, "nephewHash" => nephew_hash}) - when is_binary(uncle_hash) and is_binary(nephew_hash) do - %{nephew_hash: nephew_hash, uncle_hash: uncle_hash} + def elixir_to_params(%{"hash" => uncle_hash, "nephewHash" => nephew_hash, "index" => index}) + when is_binary(uncle_hash) and is_binary(nephew_hash) and is_integer(index) do + %{nephew_hash: nephew_hash, uncle_hash: uncle_hash, index: index} end end diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/uncles.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/uncles.ex index 6755a439be..817474faf9 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/uncles.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/uncles.ex @@ -16,14 +16,16 @@ defmodule EthereumJSONRPC.Uncles do ...> [ ...> %{ ...> "hash" => "0xe670ec64341771606e55d6b4ca35a1a6b75ee3d5145a99d05921026d15273311", - ...> "nephewHash" => "0xe52d77084cab13a4e724162bcd8c6028e5ecfaa04d091ee476e96b9958ed6b47" + ...> "nephewHash" => "0xe52d77084cab13a4e724162bcd8c6028e5ecfaa04d091ee476e96b9958ed6b47", + ...> "index" => 0 ...> } ...> ] ...> ) [ %{ uncle_hash: "0xe670ec64341771606e55d6b4ca35a1a6b75ee3d5145a99d05921026d15273311", - nephew_hash: "0xe52d77084cab13a4e724162bcd8c6028e5ecfaa04d091ee476e96b9958ed6b47" + nephew_hash: "0xe52d77084cab13a4e724162bcd8c6028e5ecfaa04d091ee476e96b9958ed6b47", + index: 0 } ] diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index 7ccf1d0606..d99bd09661 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -1479,24 +1479,23 @@ defmodule Explorer.Chain do end @doc """ - Returns a stream of all `t:Explorer.Chain.Block.t/0` `hash`es that are marked as unfetched in - `t:Explorer.Chain.Block.SecondDegreeRelation.t/0`. + Returns a stream of all blocks that are marked as unfetched in `t:Explorer.Chain.Block.SecondDegreeRelation.t/0`. + For each uncle block a `hash` of nephew block and an `index` of the block in it are returned. When a block is fetched, its uncles are transformed into `t:Explorer.Chain.Block.SecondDegreeRelation.t/0` and can be returned. Once the uncle is imported its corresponding `t:Explorer.Chain.Block.SecondDegreeRelation.t/0` `uncle_fetched_at` will be set and it won't be returned anymore. """ - @spec stream_unfetched_uncle_hashes( + @spec stream_unfetched_uncles( initial :: accumulator, - reducer :: (entry :: Hash.Full.t(), accumulator -> accumulator) + reducer :: (entry :: term(), accumulator -> accumulator) ) :: {:ok, accumulator} when accumulator: term() - def stream_unfetched_uncle_hashes(initial, reducer) when is_function(reducer, 2) do + def stream_unfetched_uncles(initial, reducer) when is_function(reducer, 2) do query = from(bsdr in Block.SecondDegreeRelation, - where: is_nil(bsdr.uncle_fetched_at), - select: bsdr.uncle_hash, - group_by: bsdr.uncle_hash + where: is_nil(bsdr.uncle_fetched_at) and not is_nil(bsdr.index), + select: [:nephew_hash, :index] ) Repo.stream_reduce(query, initial, reducer) diff --git a/apps/explorer/lib/explorer/chain/block/second_degree_relation.ex b/apps/explorer/lib/explorer/chain/block/second_degree_relation.ex index ae9b367937..b9a2af6de1 100644 --- a/apps/explorer/lib/explorer/chain/block/second_degree_relation.ex +++ b/apps/explorer/lib/explorer/chain/block/second_degree_relation.ex @@ -17,7 +17,7 @@ defmodule Explorer.Chain.Block.SecondDegreeRelation do alias Explorer.Chain.{Block, Hash} @optional_fields ~w(uncle_fetched_at)a - @required_fields ~w(nephew_hash uncle_hash)a + @required_fields ~w(nephew_hash uncle_hash index)a @allowed_fields @optional_fields ++ @required_fields @typedoc """ @@ -27,6 +27,7 @@ defmodule Explorer.Chain.Block.SecondDegreeRelation do `uncle_hash` was fetched for some other reason already. * `uncle_fetched_at` - when `t:Explorer.Chain.Block.t/0` for `uncle_hash` was confirmed as fetched. * `uncle_hash` - foreign key for `uncle`. + * `index` - index of the uncle within its nephew. Can be `nil` for blocks fetched before this field was added. """ @type t :: %__MODULE__{ @@ -34,19 +35,22 @@ defmodule Explorer.Chain.Block.SecondDegreeRelation do nephew_hash: Hash.Full.t(), uncle: %Ecto.Association.NotLoaded{} | Block.t() | nil, uncle_fetched_at: nil, - uncle_hash: Hash.Full.t() + uncle_hash: Hash.Full.t(), + index: non_neg_integer() | nil } | %__MODULE__{ nephew: %Ecto.Association.NotLoaded{} | Block.t(), nephew_hash: Hash.Full.t(), uncle: %Ecto.Association.NotLoaded{} | Block.t(), uncle_fetched_at: DateTime.t(), - uncle_hash: Hash.Full.t() + uncle_hash: Hash.Full.t(), + index: non_neg_integer() | nil } @primary_key false schema "block_second_degree_relations" do field(:uncle_fetched_at, :utc_datetime_usec) + field(:index, :integer) belongs_to(:nephew, Block, foreign_key: :nephew_hash, primary_key: true, references: :hash, type: Hash.Full) belongs_to(:uncle, Block, foreign_key: :uncle_hash, primary_key: true, references: :hash, type: Hash.Full) diff --git a/apps/explorer/lib/explorer/chain/import/runner/block/second_degree_relations.ex b/apps/explorer/lib/explorer/chain/import/runner/block/second_degree_relations.ex index 972406882b..8c8f4e26c4 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/block/second_degree_relations.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/block/second_degree_relations.ex @@ -15,7 +15,11 @@ defmodule Explorer.Chain.Import.Runner.Block.SecondDegreeRelations do @timeout 60_000 @type imported :: [ - %{required(:nephew_hash) => Hash.Full.t(), required(:uncle_hash) => Hash.Full.t()} + %{ + required(:nephew_hash) => Hash.Full.t(), + required(:uncle_hash) => Hash.Full.t(), + required(:index) => non_neg_integer() + } ] @impl Import.Runner @@ -27,8 +31,9 @@ defmodule Explorer.Chain.Import.Runner.Block.SecondDegreeRelations do @impl Import.Runner def imported_table_row do %{ - value_type: "[%{uncle_hash: Explorer.Chain.Hash.t(), nephew_hash: Explorer.Chain.Hash.t()]", - value_description: "List of maps of the `t:#{ecto_schema_module()}.t/0` `uncle_hash` and `nephew_hash`" + value_type: + "[%{uncle_hash: Explorer.Chain.Hash.t(), nephew_hash: Explorer.Chain.Hash.t(), index: non_neg_integer()]", + value_description: "List of maps of the `t:#{ecto_schema_module()}.t/0` `uncle_hash`, `nephew_hash` and `index`" } end @@ -51,7 +56,9 @@ defmodule Explorer.Chain.Import.Runner.Block.SecondDegreeRelations do @spec insert(Repo.t(), [map()], %{ optional(:on_conflict) => Import.Runner.on_conflict(), required(:timeout) => timeout - }) :: {:ok, %{nephew_hash: Hash.Full.t(), uncle_hash: Hash.Full.t()}} | {:error, [Changeset.t()]} + }) :: + {:ok, %{nephew_hash: Hash.Full.t(), uncle_hash: Hash.Full.t(), index: non_neg_integer()}} + | {:error, [Changeset.t()]} defp insert(repo, changes_list, %{timeout: timeout} = options) when is_atom(repo) and is_list(changes_list) do on_conflict = Map.get_lazy(options, :on_conflict, &default_on_conflict/0) @@ -62,7 +69,7 @@ defmodule Explorer.Chain.Import.Runner.Block.SecondDegreeRelations do conflict_target: [:nephew_hash, :uncle_hash], on_conflict: on_conflict, for: Block.SecondDegreeRelation, - returning: [:nephew_hash, :uncle_hash], + returning: [:nephew_hash, :uncle_hash, :index], timeout: timeout, # block_second_degree_relations doesn't have timestamps timestamps: %{} @@ -75,14 +82,16 @@ defmodule Explorer.Chain.Import.Runner.Block.SecondDegreeRelations do update: [ set: [ uncle_fetched_at: - fragment("LEAST(?, EXCLUDED.uncle_fetched_at)", block_second_degree_relation.uncle_fetched_at) + fragment("LEAST(?, EXCLUDED.uncle_fetched_at)", block_second_degree_relation.uncle_fetched_at), + index: fragment("EXCLUDED.index") ] ], where: fragment( - "LEAST(?, EXCLUDED.uncle_fetched_at) IS DISTINCT FROM ?", + "(LEAST(?, EXCLUDED.uncle_fetched_at), EXCLUDED.index) IS DISTINCT FROM (?, ?)", block_second_degree_relation.uncle_fetched_at, - block_second_degree_relation.uncle_fetched_at + block_second_degree_relation.uncle_fetched_at, + block_second_degree_relation.index ) ) end diff --git a/apps/explorer/priv/repo/migrations/20190421143300_add_index_to_bsdr.exs b/apps/explorer/priv/repo/migrations/20190421143300_add_index_to_bsdr.exs new file mode 100644 index 0000000000..dc8fc0422d --- /dev/null +++ b/apps/explorer/priv/repo/migrations/20190421143300_add_index_to_bsdr.exs @@ -0,0 +1,10 @@ +defmodule Explorer.Repo.Migrations.AddIndexToBsdr do + use Ecto.Migration + + def change do + alter table(:block_second_degree_relations) do + # Null for old relations without fetched index + add(:index, :integer, null: true) + end + end +end diff --git a/apps/explorer/test/explorer/chain/block/second_degree_relation_test.exs b/apps/explorer/test/explorer/chain/block/second_degree_relation_test.exs index 66f1097f16..1059b061d6 100644 --- a/apps/explorer/test/explorer/chain/block/second_degree_relation_test.exs +++ b/apps/explorer/test/explorer/chain/block/second_degree_relation_test.exs @@ -5,16 +5,21 @@ defmodule Explorer.Chain.Block.SecondDegreeRelationTest do alias Explorer.Chain.Block describe "changeset/2" do - test "requires hash and nephew_hash" do + test "requires hash, nephew_hash and index" do assert %Changeset{valid?: false} = changeset = Block.SecondDegreeRelation.changeset(%Block.SecondDegreeRelation{}, %{}) - assert changeset_errors(changeset) == %{nephew_hash: ["can't be blank"], uncle_hash: ["can't be blank"]} + assert changeset_errors(changeset) == %{ + nephew_hash: ["can't be blank"], + uncle_hash: ["can't be blank"], + index: ["can't be blank"] + } assert %Changeset{valid?: true} = Block.SecondDegreeRelation.changeset(%Block.SecondDegreeRelation{}, %{ nephew_hash: block_hash(), - uncle_hash: block_hash() + uncle_hash: block_hash(), + index: 0 }) end @@ -23,6 +28,7 @@ defmodule Explorer.Chain.Block.SecondDegreeRelationTest do Block.SecondDegreeRelation.changeset(%Block.SecondDegreeRelation{}, %{ nephew_hash: block_hash(), uncle_hash: block_hash(), + index: 0, uncle_fetched_at: DateTime.utc_now() }) end @@ -30,7 +36,7 @@ defmodule Explorer.Chain.Block.SecondDegreeRelationTest do test "enforces foreign key constraint on nephew_hash" do assert {:error, %Changeset{valid?: false} = changeset} = %Block.SecondDegreeRelation{} - |> Block.SecondDegreeRelation.changeset(%{nephew_hash: block_hash(), uncle_hash: block_hash()}) + |> Block.SecondDegreeRelation.changeset(%{nephew_hash: block_hash(), uncle_hash: block_hash(), index: 0}) |> Repo.insert() assert changeset_errors(changeset) == %{nephew_hash: ["does not exist"]} @@ -41,7 +47,7 @@ defmodule Explorer.Chain.Block.SecondDegreeRelationTest do assert {:error, %Changeset{valid?: false} = changeset} = %Block.SecondDegreeRelation{} - |> Block.SecondDegreeRelation.changeset(%{nephew_hash: nephew_hash, uncle_hash: hash}) + |> Block.SecondDegreeRelation.changeset(%{nephew_hash: nephew_hash, uncle_hash: hash, index: 0}) |> Repo.insert() assert changeset_errors(changeset) == %{uncle_hash: ["has already been taken"]} diff --git a/apps/explorer/test/explorer/chain/import_test.exs b/apps/explorer/test/explorer/chain/import_test.exs index edb20be7ef..5bff700dcb 100644 --- a/apps/explorer/test/explorer/chain/import_test.exs +++ b/apps/explorer/test/explorer/chain/import_test.exs @@ -1547,7 +1547,7 @@ defmodule Explorer.Chain.ImportTest do timeout: 1 }, block_second_degree_relations: %{ - params: [%{nephew_hash: block_hash, uncle_hash: uncle_hash}], + params: [%{nephew_hash: block_hash, uncle_hash: uncle_hash, index: 0}], timeout: 1 }, internal_transactions: %{ diff --git a/apps/explorer/test/explorer/chain_test.exs b/apps/explorer/test/explorer/chain_test.exs index 8b04eb922e..41c5afad71 100644 --- a/apps/explorer/test/explorer/chain_test.exs +++ b/apps/explorer/test/explorer/chain_test.exs @@ -1044,7 +1044,8 @@ defmodule Explorer.ChainTest do params: [ %{ nephew_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd", - uncle_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471be" + uncle_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471be", + index: 0 } ] }, @@ -3258,17 +3259,19 @@ defmodule Explorer.ChainTest do end end - describe "stream_unfetched_uncle_hashes/2" do + describe "stream_unfetched_uncles/2" do test "does not return uncle hashes where t:Explorer.Chain.Block.SecondDegreeRelation.t/0 uncle_fetched_at is not nil" do - %Block.SecondDegreeRelation{nephew: %Block{}, uncle_hash: uncle_hash} = insert(:block_second_degree_relation) + %Block.SecondDegreeRelation{nephew: %Block{}, nephew_hash: nephew_hash, index: index, uncle_hash: uncle_hash} = + insert(:block_second_degree_relation) - assert {:ok, [^uncle_hash]} = Explorer.Chain.stream_unfetched_uncle_hashes([], &[&1 | &2]) + assert {:ok, [%{nephew_hash: ^nephew_hash, index: ^index}]} = + Explorer.Chain.stream_unfetched_uncles([], &[&1 | &2]) query = from(bsdr in Block.SecondDegreeRelation, where: bsdr.uncle_hash == ^uncle_hash) assert {1, _} = Repo.update_all(query, set: [uncle_fetched_at: DateTime.utc_now()]) - assert {:ok, []} = Explorer.Chain.stream_unfetched_uncle_hashes([], &[&1 | &2]) + assert {:ok, []} = Explorer.Chain.stream_unfetched_uncles([], &[&1 | &2]) end end diff --git a/apps/explorer/test/support/factory.ex b/apps/explorer/test/support/factory.ex index a1155059ab..4c030a4db1 100644 --- a/apps/explorer/test/support/factory.ex +++ b/apps/explorer/test/support/factory.ex @@ -181,7 +181,8 @@ defmodule Explorer.Factory do def block_second_degree_relation_factory do %Block.SecondDegreeRelation{ uncle_hash: block_hash(), - nephew: build(:block) + nephew: build(:block), + index: 0 } end diff --git a/apps/indexer/README.md b/apps/indexer/README.md index ceb4ce9d0f..3821280b21 100644 --- a/apps/indexer/README.md +++ b/apps/indexer/README.md @@ -91,6 +91,7 @@ After all deployed instances get all needed data, these fetchers should be depre - `uncataloged_token_transfers`: extracts token transfers from logs, which previously weren't parsed due to unknown format - `addresses_without_codes`: forces complete refetch of blocks, which have created contract addresses without contract code - `failed_created_addresses`: forces refetch of contract code for failed transactions, which previously got incorrectly overwritten +- `uncles_without_index`: adds previously unfetched `index` field for unfetched blocks in `block_second_degree_relations` ## Memory Usage diff --git a/apps/indexer/lib/indexer/block/fetcher.ex b/apps/indexer/lib/indexer/block/fetcher.ex index 241142f4f6..ad986c9ba0 100644 --- a/apps/indexer/lib/indexer/block/fetcher.ex +++ b/apps/indexer/lib/indexer/block/fetcher.ex @@ -281,9 +281,7 @@ defmodule Indexer.Block.Fetcher do def async_import_token_balances(_), do: :ok def async_import_uncles(%{block_second_degree_relations: block_second_degree_relations}) do - block_second_degree_relations - |> Enum.map(& &1.uncle_hash) - |> UncleBlock.async_fetch_blocks() + UncleBlock.async_fetch_blocks(block_second_degree_relations) end def async_import_uncles(_), do: :ok diff --git a/apps/indexer/lib/indexer/fetcher/uncle_block.ex b/apps/indexer/lib/indexer/fetcher/uncle_block.ex index c74ee7ec89..ba36ffb2f9 100644 --- a/apps/indexer/lib/indexer/fetcher/uncle_block.ex +++ b/apps/indexer/lib/indexer/fetcher/uncle_block.ex @@ -29,17 +29,13 @@ defmodule Indexer.Fetcher.UncleBlock do ] @doc """ - Asynchronously fetches `t:Explorer.Chain.Block.t/0` for the given `hashes` and updates - `t:Explorer.Chain.Block.SecondDegreeRelation.t/0` `block_fetched_at`. + Asynchronously fetches `t:Explorer.Chain.Block.t/0` for the given `nephew_hash` and `index` + and updates `t:Explorer.Chain.Block.SecondDegreeRelation.t/0` `block_fetched_at`. """ - @spec async_fetch_blocks([Hash.Full.t()]) :: :ok - def async_fetch_blocks(block_hashes) when is_list(block_hashes) do - BufferedTask.buffer( - __MODULE__, - block_hashes - |> Enum.map(&to_string/1) - |> Enum.uniq() - ) + @spec async_fetch_blocks([%{required(:nephew_hash) => Hash.Full.t(), required(:index) => non_neg_integer()}]) :: :ok + def async_fetch_blocks(relations) when is_list(relations) do + entries = Enum.map(relations, &entry/1) + BufferedTask.buffer(__MODULE__, entries) end @doc false @@ -63,9 +59,9 @@ defmodule Indexer.Fetcher.UncleBlock do @impl BufferedTask def init(initial, reducer, _) do {:ok, final} = - Chain.stream_unfetched_uncle_hashes(initial, fn uncle_hash, acc -> - uncle_hash - |> to_string() + Chain.stream_unfetched_uncles(initial, fn uncle, acc -> + uncle + |> entry() |> reducer.(acc) end) @@ -74,31 +70,40 @@ defmodule Indexer.Fetcher.UncleBlock do @impl BufferedTask @decorate trace(name: "fetch", resource: "Indexer.Fetcher.UncleBlock.run/2", service: :indexer, tracer: Tracer) - def run(hashes, %Block.Fetcher{json_rpc_named_arguments: json_rpc_named_arguments} = block_fetcher) do - # the same block could be included as an uncle on multiple blocks, but we only want to fetch it once - unique_hashes = Enum.uniq(hashes) - - unique_hash_count = Enum.count(unique_hashes) - Logger.metadata(count: unique_hash_count) + def run(entries, %Block.Fetcher{json_rpc_named_arguments: json_rpc_named_arguments} = block_fetcher) do + entry_count = Enum.count(entries) + Logger.metadata(count: entry_count) Logger.debug("fetching") - case EthereumJSONRPC.fetch_blocks_by_hash(unique_hashes, json_rpc_named_arguments) do + entries + |> Enum.map(&entry_to_params/1) + |> EthereumJSONRPC.fetch_uncle_blocks(json_rpc_named_arguments) + |> case do {:ok, blocks} -> - run_blocks(blocks, block_fetcher, unique_hashes) + run_blocks(blocks, block_fetcher, entries) {:error, reason} -> Logger.error( fn -> ["failed to fetch: ", inspect(reason)] end, - error_count: unique_hash_count + error_count: entry_count ) - {:retry, unique_hashes} + {:retry, entries} end end + defp entry_to_params({nephew_hash_bytes, index}) when is_integer(index) do + {:ok, nephew_hash} = Hash.Full.cast(nephew_hash_bytes) + %{nephew_hash: to_string(nephew_hash), index: index} + end + + defp entry(%{nephew_hash: %Hash{bytes: nephew_hash_bytes}, index: index}) do + {nephew_hash_bytes, index} + end + defp run_blocks(%Blocks{blocks_params: []}, _, original_entries), do: {:retry, original_entries} defp run_blocks( @@ -158,9 +163,7 @@ defmodule Indexer.Fetcher.UncleBlock do # * Token.async_fetch is not called because the tokens only matter on consensus blocks # * TokenBalance.async_fetch is not called because it uses block numbers from consensus, not uncles - block_second_degree_relations - |> Enum.map(& &1.uncle_hash) - |> UncleBlock.async_fetch_blocks() + UncleBlock.async_fetch_blocks(block_second_degree_relations) ok end diff --git a/apps/indexer/lib/indexer/supervisor.ex b/apps/indexer/lib/indexer/supervisor.ex index 113aa56576..73be10f098 100644 --- a/apps/indexer/lib/indexer/supervisor.ex +++ b/apps/indexer/lib/indexer/supervisor.ex @@ -25,7 +25,8 @@ defmodule Indexer.Supervisor do alias Indexer.Temporary.{ AddressesWithoutCode, FailedCreatedAddresses, - UncatalogedTokenTransfers + UncatalogedTokenTransfers, + UnclesWithoutIndex } def child_spec([]) do @@ -129,7 +130,9 @@ defmodule Indexer.Supervisor do # Temporary workers {AddressesWithoutCode.Supervisor, [fixing_realtime_fetcher]}, {FailedCreatedAddresses.Supervisor, [json_rpc_named_arguments]}, - {UncatalogedTokenTransfers.Supervisor, [[]]} + {UncatalogedTokenTransfers.Supervisor, [[]]}, + {UnclesWithoutIndex.Supervisor, + [[json_rpc_named_arguments: json_rpc_named_arguments, memory_monitor: memory_monitor]]} ], strategy: :one_for_one ) diff --git a/apps/indexer/lib/indexer/temporary/uncles_without_index.ex b/apps/indexer/lib/indexer/temporary/uncles_without_index.ex new file mode 100644 index 0000000000..948f26b60d --- /dev/null +++ b/apps/indexer/lib/indexer/temporary/uncles_without_index.ex @@ -0,0 +1,164 @@ +defmodule Indexer.Temporary.UnclesWithoutIndex do + @moduledoc """ + Fetches `index`es for unfetched `t:Explorer.Chain.Block.SecondDegreeRelation.t/0`. + As we don't explicitly store uncle block lists for nephew blocks, we need to refetch + them completely. + """ + + use Indexer.Fetcher + use Spandex.Decorators + + require Logger + + import Ecto.Query + + alias EthereumJSONRPC.Blocks + alias Explorer.{Chain, Repo} + alias Explorer.Chain.Block.SecondDegreeRelation + alias Indexer.{BufferedTask, Tracer} + alias Indexer.Fetcher.UncleBlock + + @behaviour BufferedTask + + @defaults [ + flush_interval: :timer.seconds(3), + max_batch_size: 100, + max_concurrency: 10, + task_supervisor: Indexer.Temporary.UnclesWithoutIndex.TaskSupervisor, + metadata: [fetcher: :uncles_without_index] + ] + + @doc false + def child_spec([init_options, gen_server_options]) when is_list(init_options) do + {state, mergeable_init_options} = Keyword.pop(init_options, :json_rpc_named_arguments) + + unless state do + raise ArgumentError, + ":json_rpc_named_arguments must be provided to `#{__MODULE__}.child_spec " <> + "to allow for json_rpc calls when running." + end + + merged_init_options = + @defaults + |> Keyword.merge(mergeable_init_options) + |> Keyword.put(:state, state) + + Supervisor.child_spec({BufferedTask, [{__MODULE__, merged_init_options}, gen_server_options]}, id: __MODULE__) + end + + @impl BufferedTask + def init(initial, reducer, _) do + query = + from(bsdr in SecondDegreeRelation, + join: b in assoc(bsdr, :nephew), + where: is_nil(bsdr.index) and is_nil(bsdr.uncle_fetched_at) and b.consensus, + select: bsdr.nephew_hash, + group_by: bsdr.nephew_hash + ) + + {:ok, final} = + Repo.stream_reduce(query, initial, fn nephew_hash, acc -> + nephew_hash + |> to_string() + |> reducer.(acc) + end) + + final + end + + @impl BufferedTask + @decorate trace(name: "fetch", resource: "Indexer.Fetcher.UncleBlock.run/2", service: :indexer, tracer: Tracer) + def run(hashes, json_rpc_named_arguments) do + hash_count = Enum.count(hashes) + Logger.metadata(count: hash_count) + + Logger.debug("fetching") + + case EthereumJSONRPC.fetch_blocks_by_hash(hashes, json_rpc_named_arguments) do + {:ok, blocks} -> + run_blocks(blocks, hashes) + + {:error, reason} -> + Logger.error( + fn -> + ["failed to fetch: ", inspect(reason)] + end, + error_count: hash_count + ) + + {:retry, hashes} + end + end + + defp run_blocks(%Blocks{blocks_params: []}, original_entries), do: {:retry, original_entries} + + defp run_blocks( + %Blocks{block_second_degree_relations_params: block_second_degree_relations_params, errors: errors}, + original_entries + ) do + case Chain.import(%{block_second_degree_relations: %{params: block_second_degree_relations_params}}) do + {:ok, %{block_second_degree_relations: block_second_degree_relations}} -> + UncleBlock.async_fetch_blocks(block_second_degree_relations) + + retry(errors) + + {:error, step, failed_value, _changes_so_far} -> + Logger.error(fn -> ["failed to import: ", inspect(failed_value)] end, + step: step, + error_count: Enum.count(original_entries) + ) + + {:retry, original_entries} + end + end + + defp retry([]), do: :ok + + defp retry(errors) when is_list(errors) do + retried_entries = errors_to_entries(errors) + loggable_errors = loggable_errors(errors) + loggable_error_count = Enum.count(loggable_errors) + + unless loggable_error_count == 0 do + Logger.error( + fn -> + [ + "failed to fetch: ", + errors_to_iodata(loggable_errors) + ] + end, + error_count: loggable_error_count + ) + end + + {:retry, retried_entries} + end + + defp loggable_errors(errors) when is_list(errors) do + Enum.filter(errors, fn + %{code: 404, message: "Not Found"} -> false + _ -> true + end) + end + + defp errors_to_entries(errors) when is_list(errors) do + Enum.map(errors, &error_to_entry/1) + end + + defp error_to_entry(%{data: %{hash: hash}}) when is_binary(hash), do: hash + + defp errors_to_iodata(errors) when is_list(errors) do + errors_to_iodata(errors, []) + end + + defp errors_to_iodata([], iodata), do: iodata + + defp errors_to_iodata([error | errors], iodata) do + errors_to_iodata(errors, [iodata | error_to_iodata(error)]) + end + + defp error_to_iodata(%{code: code, message: message, data: %{hash: hash}}) + when is_integer(code) and is_binary(message) and is_binary(hash) do + [hash, ": (", to_string(code), ") ", message, ?\n] + end +end diff --git a/apps/indexer/test/indexer/block/catchup/fetcher_test.exs b/apps/indexer/test/indexer/block/catchup/fetcher_test.exs index 9588e0423d..6230552c3f 100644 --- a/apps/indexer/test/indexer/block/catchup/fetcher_test.exs +++ b/apps/indexer/test/indexer/block/catchup/fetcher_test.exs @@ -7,6 +7,7 @@ defmodule Indexer.Block.Catchup.FetcherTest do alias Explorer.Chain alias Explorer.Chain.Block.Reward + alias Explorer.Chain.Hash alias Indexer.Block alias Indexer.Block.Catchup.Fetcher alias Indexer.Fetcher.{BlockReward, CoinBalance, InternalTransaction, Token, TokenBalance, UncleBlock} @@ -51,7 +52,10 @@ defmodule Indexer.Block.Catchup.FetcherTest do Process.register(pid, UncleBlock) - nephew_hash = block_hash() |> to_string() + nephew_hash_data = block_hash() + %Hash{bytes: nephew_hash_bytes} = nephew_hash_data + nephew_hash = nephew_hash_data |> to_string() + nephew_index = 0 uncle_hash = block_hash() |> to_string() miner_hash = address_hash() |> to_string() block_number = 0 @@ -96,7 +100,8 @@ defmodule Indexer.Block.Catchup.FetcherTest do params: [ %{ nephew_hash: nephew_hash, - uncle_hash: uncle_hash + uncle_hash: uncle_hash, + index: nephew_index } ] }, @@ -113,7 +118,7 @@ defmodule Indexer.Block.Catchup.FetcherTest do } }) - assert_receive {:uncles, [^uncle_hash]} + assert_receive {:uncles, [{^nephew_hash_bytes, ^nephew_index}]} end end diff --git a/apps/indexer/test/indexer/fetcher/uncle_block_test.exs b/apps/indexer/test/indexer/fetcher/uncle_block_test.exs index 0f8bb75f02..224349d781 100644 --- a/apps/indexer/test/indexer/fetcher/uncle_block_test.exs +++ b/apps/indexer/test/indexer/fetcher/uncle_block_test.exs @@ -4,6 +4,8 @@ defmodule Indexer.Fetcher.UncleBlockTest do use EthereumJSONRPC.Case, async: false use Explorer.DataCase + import EthereumJSONRPC, only: [integer_to_quantity: 1] + alias Explorer.Chain alias Indexer.Block alias Indexer.Fetcher.UncleBlock @@ -43,16 +45,30 @@ defmodule Indexer.Fetcher.UncleBlockTest do describe "init/1" do test "fetched unfetched uncle hashes", %{json_rpc_named_arguments: json_rpc_named_arguments} do - assert %Chain.Block.SecondDegreeRelation{nephew_hash: nephew_hash, uncle_hash: uncle_hash, uncle: nil} = + assert %Chain.Block.SecondDegreeRelation{ + nephew_hash: nephew_hash, + uncle_hash: uncle_hash, + index: index, + uncle: nil + } = :block_second_degree_relation |> insert() |> Repo.preload([:nephew, :uncle]) + nephew_hash_data = to_string(nephew_hash) uncle_hash_data = to_string(uncle_hash) uncle_uncle_hash_data = to_string(block_hash()) + index_data = integer_to_quantity(index) EthereumJSONRPC.Mox - |> expect(:json_rpc, fn [%{id: id, method: "eth_getBlockByHash", params: [^uncle_hash_data, true]}], _ -> + |> expect(:json_rpc, fn [ + %{ + id: id, + method: "eth_getUncleByBlockHashAndIndex", + params: [^nephew_hash_data, ^index_data] + } + ], + _ -> number_quantity = "0x0" {:ok, From ea7add399adbba08d41455e4bb3a0bb8261f334f Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Mon, 22 Apr 2019 17:53:13 +0300 Subject: [PATCH 08/12] Divide chains by Mainnet/Testnet in menu --- apps/block_scout_web/assets/css/components/_navbar.scss | 4 ++++ .../lib/block_scout_web/templates/layout/_topnav.html.eex | 7 ++++++- .../lib/block_scout_web/views/layout_view.ex | 1 + 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/apps/block_scout_web/assets/css/components/_navbar.scss b/apps/block_scout_web/assets/css/components/_navbar.scss index 667414d98a..75b5efc218 100644 --- a/apps/block_scout_web/assets/css/components/_navbar.scss +++ b/apps/block_scout_web/assets/css/components/_navbar.scss @@ -158,6 +158,10 @@ background-color: $tertiary; color: $white; } + + &.header { + font-weight: bold; + } } .add-border { diff --git a/apps/block_scout_web/lib/block_scout_web/templates/layout/_topnav.html.eex b/apps/block_scout_web/lib/block_scout_web/templates/layout/_topnav.html.eex index f000e09111..336ba3c208 100644 --- a/apps/block_scout_web/lib/block_scout_web/templates/layout/_topnav.html.eex +++ b/apps/block_scout_web/lib/block_scout_web/templates/layout/_topnav.html.eex @@ -120,7 +120,12 @@ <%= subnetwork_title() %>
- <%= for %{url: url, title: title} <- other_networks() do %> + Mainnet + <%= for %{url: url, title: title} <- main_nets() do %> + <%= title %> + <% end %> + Testnet + <%= for %{url: url, title: title} <- test_nets() do %> <%= title %> <% end %>
diff --git a/apps/block_scout_web/lib/block_scout_web/views/layout_view.ex b/apps/block_scout_web/lib/block_scout_web/views/layout_view.ex index 1e76ecb3f7..fcde8d5371 100644 --- a/apps/block_scout_web/lib/block_scout_web/views/layout_view.ex +++ b/apps/block_scout_web/lib/block_scout_web/views/layout_view.ex @@ -95,6 +95,7 @@ defmodule BlockScoutWeb.LayoutView do |> Enum.reject(fn %{title: title} -> title == subnetwork_title() end) + |> Enum.sort() end def main_nets do From de79defb4c9c204261f5bf8e1070343e3d41fd26 Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Mon, 22 Apr 2019 17:59:14 +0300 Subject: [PATCH 09/12] Divider between prod/test in menu --- apps/block_scout_web/assets/css/components/_navbar.scss | 4 ++++ .../lib/block_scout_web/templates/layout/_topnav.html.eex | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/apps/block_scout_web/assets/css/components/_navbar.scss b/apps/block_scout_web/assets/css/components/_navbar.scss index 75b5efc218..685fbb2fec 100644 --- a/apps/block_scout_web/assets/css/components/_navbar.scss +++ b/apps/block_scout_web/assets/css/components/_navbar.scss @@ -161,6 +161,10 @@ &.header { font-weight: bold; + + &.division { + border-top: 1px solid rgb(183, 185, 184); + } } } diff --git a/apps/block_scout_web/lib/block_scout_web/templates/layout/_topnav.html.eex b/apps/block_scout_web/lib/block_scout_web/templates/layout/_topnav.html.eex index 336ba3c208..f6eb3730e4 100644 --- a/apps/block_scout_web/lib/block_scout_web/templates/layout/_topnav.html.eex +++ b/apps/block_scout_web/lib/block_scout_web/templates/layout/_topnav.html.eex @@ -124,7 +124,7 @@ <%= for %{url: url, title: title} <- main_nets() do %> <%= title %> <% end %> - Testnet + Testnet <%= for %{url: url, title: title} <- test_nets() do %> <%= title %> <% end %> From 2131384d2bf4e4ecf7384ea2cfee645d499722bc Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Mon, 22 Apr 2019 18:01:54 +0300 Subject: [PATCH 10/12] Update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index aab08669f8..8cc85afd75 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ ### Chore + - [#1804](https://github.com/poanetwork/blockscout/pull/1804) - (Chore) Divide chains by Mainnet/Testnet in menu - [#1783](https://github.com/poanetwork/blockscout/pull/1783) - Update README with the chains that use Blockscout - [#1780](https://github.com/poanetwork/blockscout/pull/1780) - Update link to the Github repo in the footer - [#1757](https://github.com/poanetwork/blockscout/pull/1757) - Change twitter acc link to official Blockscout acc twitter From 49061dd9e97c5e49566b3c3627d697d806c8f304 Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Mon, 22 Apr 2019 18:21:54 +0300 Subject: [PATCH 11/12] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ec4d79e2f..1c02dd8506 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ - [#1803](https://github.com/poanetwork/blockscout/pull/1803) - use coinmarketcap for total_supply by default - [#1802](https://github.com/poanetwork/blockscout/pull/1802) - make coinmarketcap's number of pages configurable - [#1799](https://github.com/poanetwork/blockscout/pull/1799) - Use eth_getUncleByBlockHashAndIndex for uncle block fetching + - [#1531](https://github.com/poanetwork/blockscout/pull/1531) - docker: fix dockerFile for secp256k1 building ### Chore From 4862f9dd407957766579b773e62c191047db8a8d Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Mon, 22 Apr 2019 18:50:07 +0300 Subject: [PATCH 12/12] Update CHANGELOG.md --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c02dd8506..9f88ff0a14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ ### Features +### Fixes + +### Chore + + +## 1.3.10-beta + +### Features + - [#1739](https://github.com/poanetwork/blockscout/pull/1739) - highlight decompiled source code - [#1696](https://github.com/poanetwork/blockscout/pull/1696) - full-text search by tokens - [#1742](https://github.com/poanetwork/blockscout/pull/1742) - Support RSK