diff --git a/CHANGELOG.md b/CHANGELOG.md index b9f18ede21..9ad6ec7098 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ - [#1793](https://github.com/poanetwork/blockscout/pull/1793) - fix top nav autocomplete - [#1795](https://github.com/poanetwork/blockscout/pull/1795) - fix line numbers for decompiled contracts - [#1802](https://github.com/poanetwork/blockscout/pull/1802) - make coinmarketcap's number of pages configurable + - [#1799](https://github.com/poanetwork/blockscout/pull/1799) - Use eth_getUncleByBlockHashAndIndex for uncle block fetching ### Chore diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc.ex index 33d3abd31b..24fc593d85 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc.ex @@ -52,6 +52,11 @@ defmodule EthereumJSONRPC do """ @type block_number :: non_neg_integer() + @typedoc """ + Reference to an uncle block by nephew block's `hash` and `index` in it. + """ + @type nephew_index :: %{required(:nephew_hash) => String.t(), required(:index) => non_neg_integer()} + @typedoc """ Binary data encoded as a single hexadecimal number in a `String.t` """ @@ -235,6 +240,15 @@ defmodule EthereumJSONRPC do |> fetch_blocks_by_params(&Block.ByNumber.request/1, json_rpc_named_arguments) end + @doc """ + Fetches uncle blocks by nephew hashes and indices. + """ + @spec fetch_uncle_blocks([nephew_index()], json_rpc_named_arguments) :: {:ok, Blocks.t()} | {:error, reason :: term} + def fetch_uncle_blocks(blocks, json_rpc_named_arguments) do + blocks + |> fetch_blocks_by_params(&Block.ByNephew.request/1, json_rpc_named_arguments) + end + @doc """ Fetches block number by `t:tag/0`. diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/block.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/block.ex index 3aa6eed029..b2f3a87fcd 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/block.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/block.ex @@ -356,14 +356,17 @@ defmodule EthereumJSONRPC.Block do [ %{ "hash" => "0xe670ec64341771606e55d6b4ca35a1a6b75ee3d5145a99d05921026d15273311", - "nephewHash" => "0xe52d77084cab13a4e724162bcd8c6028e5ecfaa04d091ee476e96b9958ed6b47" + "nephewHash" => "0xe52d77084cab13a4e724162bcd8c6028e5ecfaa04d091ee476e96b9958ed6b47", + "index" => 0 } ] """ @spec elixir_to_uncles(elixir) :: Uncles.elixir() def elixir_to_uncles(%{"hash" => nephew_hash, "uncles" => uncles}) do - Enum.map(uncles, &%{"hash" => &1, "nephewHash" => nephew_hash}) + uncles + |> Enum.with_index() + |> Enum.map(fn {uncle_hash, index} -> %{"hash" => uncle_hash, "nephewHash" => nephew_hash, "index" => index} end) end @doc """ diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/block/by_nephew.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/block/by_nephew.ex new file mode 100644 index 0000000000..56e8393343 --- /dev/null +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/block/by_nephew.ex @@ -0,0 +1,15 @@ +defmodule EthereumJSONRPC.Block.ByNephew do + @moduledoc """ + Block format as returned by [`eth_getUncleByBlockHashAndIndex`](https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_getUncleByBlockHashAndIndex) + """ + + import EthereumJSONRPC, only: [integer_to_quantity: 1] + + def request(%{id: id, nephew_hash: nephew_hash, index: index}) do + EthereumJSONRPC.request(%{ + id: id, + method: "eth_getUncleByBlockHashAndIndex", + params: [nephew_hash, integer_to_quantity(index)] + }) + end +end diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/blocks.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/blocks.ex index 22bb74faf6..dc1740a4aa 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/blocks.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/blocks.ex @@ -260,7 +260,8 @@ defmodule EthereumJSONRPC.Blocks do [ %{ "hash" => "0xe670ec64341771606e55d6b4ca35a1a6b75ee3d5145a99d05921026d15273311", - "nephewHash" => "0xe52d77084cab13a4e724162bcd8c6028e5ecfaa04d091ee476e96b9958ed6b47" + "nephewHash" => "0xe52d77084cab13a4e724162bcd8c6028e5ecfaa04d091ee476e96b9958ed6b47", + "index" => 0 } ] diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/uncle.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/uncle.ex index 871f0daa7d..fb0a6a397e 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/uncle.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/uncle.ex @@ -6,15 +6,16 @@ defmodule EthereumJSONRPC.Uncle do chain. """ - @type elixir :: %{String.t() => EthereumJSONRPC.hash()} + @type elixir :: %{String.t() => EthereumJSONRPC.hash() | non_neg_integer()} @typedoc """ * `"hash"` - the hash of the uncle block. * `"nephewHash"` - the hash of the nephew block that included `"hash` as an uncle. + * `"index"` - the index of the uncle block within the nephew block. """ @type t :: %{String.t() => EthereumJSONRPC.hash()} - @type params :: %{nephew_hash: EthereumJSONRPC.hash(), uncle_hash: EthereumJSONRPC.hash()} + @type params :: %{nephew_hash: EthereumJSONRPC.hash(), uncle_hash: EthereumJSONRPC.hash(), index: non_neg_integer()} @doc """ Converts each entry in `t:elixir/0` to `t:params/0` used in `Explorer.Chain.Uncle.changeset/2`. @@ -22,18 +23,20 @@ defmodule EthereumJSONRPC.Uncle do iex> EthereumJSONRPC.Uncle.elixir_to_params( ...> %{ ...> "hash" => "0xe670ec64341771606e55d6b4ca35a1a6b75ee3d5145a99d05921026d15273311", - ...> "nephewHash" => "0xe52d77084cab13a4e724162bcd8c6028e5ecfaa04d091ee476e96b9958ed6b47" + ...> "nephewHash" => "0xe52d77084cab13a4e724162bcd8c6028e5ecfaa04d091ee476e96b9958ed6b47", + ...> "index" => 0 ...> } ...> ) %{ nephew_hash: "0xe52d77084cab13a4e724162bcd8c6028e5ecfaa04d091ee476e96b9958ed6b47", - uncle_hash: "0xe670ec64341771606e55d6b4ca35a1a6b75ee3d5145a99d05921026d15273311" + uncle_hash: "0xe670ec64341771606e55d6b4ca35a1a6b75ee3d5145a99d05921026d15273311", + index: 0 } """ @spec elixir_to_params(elixir) :: params - def elixir_to_params(%{"hash" => uncle_hash, "nephewHash" => nephew_hash}) - when is_binary(uncle_hash) and is_binary(nephew_hash) do - %{nephew_hash: nephew_hash, uncle_hash: uncle_hash} + def elixir_to_params(%{"hash" => uncle_hash, "nephewHash" => nephew_hash, "index" => index}) + when is_binary(uncle_hash) and is_binary(nephew_hash) and is_integer(index) do + %{nephew_hash: nephew_hash, uncle_hash: uncle_hash, index: index} end end diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/uncles.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/uncles.ex index 6755a439be..817474faf9 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/uncles.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/uncles.ex @@ -16,14 +16,16 @@ defmodule EthereumJSONRPC.Uncles do ...> [ ...> %{ ...> "hash" => "0xe670ec64341771606e55d6b4ca35a1a6b75ee3d5145a99d05921026d15273311", - ...> "nephewHash" => "0xe52d77084cab13a4e724162bcd8c6028e5ecfaa04d091ee476e96b9958ed6b47" + ...> "nephewHash" => "0xe52d77084cab13a4e724162bcd8c6028e5ecfaa04d091ee476e96b9958ed6b47", + ...> "index" => 0 ...> } ...> ] ...> ) [ %{ uncle_hash: "0xe670ec64341771606e55d6b4ca35a1a6b75ee3d5145a99d05921026d15273311", - nephew_hash: "0xe52d77084cab13a4e724162bcd8c6028e5ecfaa04d091ee476e96b9958ed6b47" + nephew_hash: "0xe52d77084cab13a4e724162bcd8c6028e5ecfaa04d091ee476e96b9958ed6b47", + index: 0 } ] diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index 7ccf1d0606..d99bd09661 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -1479,24 +1479,23 @@ defmodule Explorer.Chain do end @doc """ - Returns a stream of all `t:Explorer.Chain.Block.t/0` `hash`es that are marked as unfetched in - `t:Explorer.Chain.Block.SecondDegreeRelation.t/0`. + Returns a stream of all blocks that are marked as unfetched in `t:Explorer.Chain.Block.SecondDegreeRelation.t/0`. + For each uncle block a `hash` of nephew block and an `index` of the block in it are returned. When a block is fetched, its uncles are transformed into `t:Explorer.Chain.Block.SecondDegreeRelation.t/0` and can be returned. Once the uncle is imported its corresponding `t:Explorer.Chain.Block.SecondDegreeRelation.t/0` `uncle_fetched_at` will be set and it won't be returned anymore. """ - @spec stream_unfetched_uncle_hashes( + @spec stream_unfetched_uncles( initial :: accumulator, - reducer :: (entry :: Hash.Full.t(), accumulator -> accumulator) + reducer :: (entry :: term(), accumulator -> accumulator) ) :: {:ok, accumulator} when accumulator: term() - def stream_unfetched_uncle_hashes(initial, reducer) when is_function(reducer, 2) do + def stream_unfetched_uncles(initial, reducer) when is_function(reducer, 2) do query = from(bsdr in Block.SecondDegreeRelation, - where: is_nil(bsdr.uncle_fetched_at), - select: bsdr.uncle_hash, - group_by: bsdr.uncle_hash + where: is_nil(bsdr.uncle_fetched_at) and not is_nil(bsdr.index), + select: [:nephew_hash, :index] ) Repo.stream_reduce(query, initial, reducer) diff --git a/apps/explorer/lib/explorer/chain/block/second_degree_relation.ex b/apps/explorer/lib/explorer/chain/block/second_degree_relation.ex index ae9b367937..b9a2af6de1 100644 --- a/apps/explorer/lib/explorer/chain/block/second_degree_relation.ex +++ b/apps/explorer/lib/explorer/chain/block/second_degree_relation.ex @@ -17,7 +17,7 @@ defmodule Explorer.Chain.Block.SecondDegreeRelation do alias Explorer.Chain.{Block, Hash} @optional_fields ~w(uncle_fetched_at)a - @required_fields ~w(nephew_hash uncle_hash)a + @required_fields ~w(nephew_hash uncle_hash index)a @allowed_fields @optional_fields ++ @required_fields @typedoc """ @@ -27,6 +27,7 @@ defmodule Explorer.Chain.Block.SecondDegreeRelation do `uncle_hash` was fetched for some other reason already. * `uncle_fetched_at` - when `t:Explorer.Chain.Block.t/0` for `uncle_hash` was confirmed as fetched. * `uncle_hash` - foreign key for `uncle`. + * `index` - index of the uncle within its nephew. Can be `nil` for blocks fetched before this field was added. """ @type t :: %__MODULE__{ @@ -34,19 +35,22 @@ defmodule Explorer.Chain.Block.SecondDegreeRelation do nephew_hash: Hash.Full.t(), uncle: %Ecto.Association.NotLoaded{} | Block.t() | nil, uncle_fetched_at: nil, - uncle_hash: Hash.Full.t() + uncle_hash: Hash.Full.t(), + index: non_neg_integer() | nil } | %__MODULE__{ nephew: %Ecto.Association.NotLoaded{} | Block.t(), nephew_hash: Hash.Full.t(), uncle: %Ecto.Association.NotLoaded{} | Block.t(), uncle_fetched_at: DateTime.t(), - uncle_hash: Hash.Full.t() + uncle_hash: Hash.Full.t(), + index: non_neg_integer() | nil } @primary_key false schema "block_second_degree_relations" do field(:uncle_fetched_at, :utc_datetime_usec) + field(:index, :integer) belongs_to(:nephew, Block, foreign_key: :nephew_hash, primary_key: true, references: :hash, type: Hash.Full) belongs_to(:uncle, Block, foreign_key: :uncle_hash, primary_key: true, references: :hash, type: Hash.Full) diff --git a/apps/explorer/lib/explorer/chain/import/runner/block/second_degree_relations.ex b/apps/explorer/lib/explorer/chain/import/runner/block/second_degree_relations.ex index 972406882b..8c8f4e26c4 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/block/second_degree_relations.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/block/second_degree_relations.ex @@ -15,7 +15,11 @@ defmodule Explorer.Chain.Import.Runner.Block.SecondDegreeRelations do @timeout 60_000 @type imported :: [ - %{required(:nephew_hash) => Hash.Full.t(), required(:uncle_hash) => Hash.Full.t()} + %{ + required(:nephew_hash) => Hash.Full.t(), + required(:uncle_hash) => Hash.Full.t(), + required(:index) => non_neg_integer() + } ] @impl Import.Runner @@ -27,8 +31,9 @@ defmodule Explorer.Chain.Import.Runner.Block.SecondDegreeRelations do @impl Import.Runner def imported_table_row do %{ - value_type: "[%{uncle_hash: Explorer.Chain.Hash.t(), nephew_hash: Explorer.Chain.Hash.t()]", - value_description: "List of maps of the `t:#{ecto_schema_module()}.t/0` `uncle_hash` and `nephew_hash`" + value_type: + "[%{uncle_hash: Explorer.Chain.Hash.t(), nephew_hash: Explorer.Chain.Hash.t(), index: non_neg_integer()]", + value_description: "List of maps of the `t:#{ecto_schema_module()}.t/0` `uncle_hash`, `nephew_hash` and `index`" } end @@ -51,7 +56,9 @@ defmodule Explorer.Chain.Import.Runner.Block.SecondDegreeRelations do @spec insert(Repo.t(), [map()], %{ optional(:on_conflict) => Import.Runner.on_conflict(), required(:timeout) => timeout - }) :: {:ok, %{nephew_hash: Hash.Full.t(), uncle_hash: Hash.Full.t()}} | {:error, [Changeset.t()]} + }) :: + {:ok, %{nephew_hash: Hash.Full.t(), uncle_hash: Hash.Full.t(), index: non_neg_integer()}} + | {:error, [Changeset.t()]} defp insert(repo, changes_list, %{timeout: timeout} = options) when is_atom(repo) and is_list(changes_list) do on_conflict = Map.get_lazy(options, :on_conflict, &default_on_conflict/0) @@ -62,7 +69,7 @@ defmodule Explorer.Chain.Import.Runner.Block.SecondDegreeRelations do conflict_target: [:nephew_hash, :uncle_hash], on_conflict: on_conflict, for: Block.SecondDegreeRelation, - returning: [:nephew_hash, :uncle_hash], + returning: [:nephew_hash, :uncle_hash, :index], timeout: timeout, # block_second_degree_relations doesn't have timestamps timestamps: %{} @@ -75,14 +82,16 @@ defmodule Explorer.Chain.Import.Runner.Block.SecondDegreeRelations do update: [ set: [ uncle_fetched_at: - fragment("LEAST(?, EXCLUDED.uncle_fetched_at)", block_second_degree_relation.uncle_fetched_at) + fragment("LEAST(?, EXCLUDED.uncle_fetched_at)", block_second_degree_relation.uncle_fetched_at), + index: fragment("EXCLUDED.index") ] ], where: fragment( - "LEAST(?, EXCLUDED.uncle_fetched_at) IS DISTINCT FROM ?", + "(LEAST(?, EXCLUDED.uncle_fetched_at), EXCLUDED.index) IS DISTINCT FROM (?, ?)", block_second_degree_relation.uncle_fetched_at, - block_second_degree_relation.uncle_fetched_at + block_second_degree_relation.uncle_fetched_at, + block_second_degree_relation.index ) ) end diff --git a/apps/explorer/priv/repo/migrations/20190421143300_add_index_to_bsdr.exs b/apps/explorer/priv/repo/migrations/20190421143300_add_index_to_bsdr.exs new file mode 100644 index 0000000000..dc8fc0422d --- /dev/null +++ b/apps/explorer/priv/repo/migrations/20190421143300_add_index_to_bsdr.exs @@ -0,0 +1,10 @@ +defmodule Explorer.Repo.Migrations.AddIndexToBsdr do + use Ecto.Migration + + def change do + alter table(:block_second_degree_relations) do + # Null for old relations without fetched index + add(:index, :integer, null: true) + end + end +end diff --git a/apps/explorer/test/explorer/chain/block/second_degree_relation_test.exs b/apps/explorer/test/explorer/chain/block/second_degree_relation_test.exs index 66f1097f16..1059b061d6 100644 --- a/apps/explorer/test/explorer/chain/block/second_degree_relation_test.exs +++ b/apps/explorer/test/explorer/chain/block/second_degree_relation_test.exs @@ -5,16 +5,21 @@ defmodule Explorer.Chain.Block.SecondDegreeRelationTest do alias Explorer.Chain.Block describe "changeset/2" do - test "requires hash and nephew_hash" do + test "requires hash, nephew_hash and index" do assert %Changeset{valid?: false} = changeset = Block.SecondDegreeRelation.changeset(%Block.SecondDegreeRelation{}, %{}) - assert changeset_errors(changeset) == %{nephew_hash: ["can't be blank"], uncle_hash: ["can't be blank"]} + assert changeset_errors(changeset) == %{ + nephew_hash: ["can't be blank"], + uncle_hash: ["can't be blank"], + index: ["can't be blank"] + } assert %Changeset{valid?: true} = Block.SecondDegreeRelation.changeset(%Block.SecondDegreeRelation{}, %{ nephew_hash: block_hash(), - uncle_hash: block_hash() + uncle_hash: block_hash(), + index: 0 }) end @@ -23,6 +28,7 @@ defmodule Explorer.Chain.Block.SecondDegreeRelationTest do Block.SecondDegreeRelation.changeset(%Block.SecondDegreeRelation{}, %{ nephew_hash: block_hash(), uncle_hash: block_hash(), + index: 0, uncle_fetched_at: DateTime.utc_now() }) end @@ -30,7 +36,7 @@ defmodule Explorer.Chain.Block.SecondDegreeRelationTest do test "enforces foreign key constraint on nephew_hash" do assert {:error, %Changeset{valid?: false} = changeset} = %Block.SecondDegreeRelation{} - |> Block.SecondDegreeRelation.changeset(%{nephew_hash: block_hash(), uncle_hash: block_hash()}) + |> Block.SecondDegreeRelation.changeset(%{nephew_hash: block_hash(), uncle_hash: block_hash(), index: 0}) |> Repo.insert() assert changeset_errors(changeset) == %{nephew_hash: ["does not exist"]} @@ -41,7 +47,7 @@ defmodule Explorer.Chain.Block.SecondDegreeRelationTest do assert {:error, %Changeset{valid?: false} = changeset} = %Block.SecondDegreeRelation{} - |> Block.SecondDegreeRelation.changeset(%{nephew_hash: nephew_hash, uncle_hash: hash}) + |> Block.SecondDegreeRelation.changeset(%{nephew_hash: nephew_hash, uncle_hash: hash, index: 0}) |> Repo.insert() assert changeset_errors(changeset) == %{uncle_hash: ["has already been taken"]} diff --git a/apps/explorer/test/explorer/chain/import_test.exs b/apps/explorer/test/explorer/chain/import_test.exs index edb20be7ef..5bff700dcb 100644 --- a/apps/explorer/test/explorer/chain/import_test.exs +++ b/apps/explorer/test/explorer/chain/import_test.exs @@ -1547,7 +1547,7 @@ defmodule Explorer.Chain.ImportTest do timeout: 1 }, block_second_degree_relations: %{ - params: [%{nephew_hash: block_hash, uncle_hash: uncle_hash}], + params: [%{nephew_hash: block_hash, uncle_hash: uncle_hash, index: 0}], timeout: 1 }, internal_transactions: %{ diff --git a/apps/explorer/test/explorer/chain_test.exs b/apps/explorer/test/explorer/chain_test.exs index 8b04eb922e..41c5afad71 100644 --- a/apps/explorer/test/explorer/chain_test.exs +++ b/apps/explorer/test/explorer/chain_test.exs @@ -1044,7 +1044,8 @@ defmodule Explorer.ChainTest do params: [ %{ nephew_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd", - uncle_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471be" + uncle_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471be", + index: 0 } ] }, @@ -3258,17 +3259,19 @@ defmodule Explorer.ChainTest do end end - describe "stream_unfetched_uncle_hashes/2" do + describe "stream_unfetched_uncles/2" do test "does not return uncle hashes where t:Explorer.Chain.Block.SecondDegreeRelation.t/0 uncle_fetched_at is not nil" do - %Block.SecondDegreeRelation{nephew: %Block{}, uncle_hash: uncle_hash} = insert(:block_second_degree_relation) + %Block.SecondDegreeRelation{nephew: %Block{}, nephew_hash: nephew_hash, index: index, uncle_hash: uncle_hash} = + insert(:block_second_degree_relation) - assert {:ok, [^uncle_hash]} = Explorer.Chain.stream_unfetched_uncle_hashes([], &[&1 | &2]) + assert {:ok, [%{nephew_hash: ^nephew_hash, index: ^index}]} = + Explorer.Chain.stream_unfetched_uncles([], &[&1 | &2]) query = from(bsdr in Block.SecondDegreeRelation, where: bsdr.uncle_hash == ^uncle_hash) assert {1, _} = Repo.update_all(query, set: [uncle_fetched_at: DateTime.utc_now()]) - assert {:ok, []} = Explorer.Chain.stream_unfetched_uncle_hashes([], &[&1 | &2]) + assert {:ok, []} = Explorer.Chain.stream_unfetched_uncles([], &[&1 | &2]) end end diff --git a/apps/explorer/test/support/factory.ex b/apps/explorer/test/support/factory.ex index a1155059ab..4c030a4db1 100644 --- a/apps/explorer/test/support/factory.ex +++ b/apps/explorer/test/support/factory.ex @@ -181,7 +181,8 @@ defmodule Explorer.Factory do def block_second_degree_relation_factory do %Block.SecondDegreeRelation{ uncle_hash: block_hash(), - nephew: build(:block) + nephew: build(:block), + index: 0 } end diff --git a/apps/indexer/README.md b/apps/indexer/README.md index ceb4ce9d0f..3821280b21 100644 --- a/apps/indexer/README.md +++ b/apps/indexer/README.md @@ -91,6 +91,7 @@ After all deployed instances get all needed data, these fetchers should be depre - `uncataloged_token_transfers`: extracts token transfers from logs, which previously weren't parsed due to unknown format - `addresses_without_codes`: forces complete refetch of blocks, which have created contract addresses without contract code - `failed_created_addresses`: forces refetch of contract code for failed transactions, which previously got incorrectly overwritten +- `uncles_without_index`: adds previously unfetched `index` field for unfetched blocks in `block_second_degree_relations` ## Memory Usage diff --git a/apps/indexer/lib/indexer/block/fetcher.ex b/apps/indexer/lib/indexer/block/fetcher.ex index 241142f4f6..ad986c9ba0 100644 --- a/apps/indexer/lib/indexer/block/fetcher.ex +++ b/apps/indexer/lib/indexer/block/fetcher.ex @@ -281,9 +281,7 @@ defmodule Indexer.Block.Fetcher do def async_import_token_balances(_), do: :ok def async_import_uncles(%{block_second_degree_relations: block_second_degree_relations}) do - block_second_degree_relations - |> Enum.map(& &1.uncle_hash) - |> UncleBlock.async_fetch_blocks() + UncleBlock.async_fetch_blocks(block_second_degree_relations) end def async_import_uncles(_), do: :ok diff --git a/apps/indexer/lib/indexer/fetcher/uncle_block.ex b/apps/indexer/lib/indexer/fetcher/uncle_block.ex index c74ee7ec89..ba36ffb2f9 100644 --- a/apps/indexer/lib/indexer/fetcher/uncle_block.ex +++ b/apps/indexer/lib/indexer/fetcher/uncle_block.ex @@ -29,17 +29,13 @@ defmodule Indexer.Fetcher.UncleBlock do ] @doc """ - Asynchronously fetches `t:Explorer.Chain.Block.t/0` for the given `hashes` and updates - `t:Explorer.Chain.Block.SecondDegreeRelation.t/0` `block_fetched_at`. + Asynchronously fetches `t:Explorer.Chain.Block.t/0` for the given `nephew_hash` and `index` + and updates `t:Explorer.Chain.Block.SecondDegreeRelation.t/0` `block_fetched_at`. """ - @spec async_fetch_blocks([Hash.Full.t()]) :: :ok - def async_fetch_blocks(block_hashes) when is_list(block_hashes) do - BufferedTask.buffer( - __MODULE__, - block_hashes - |> Enum.map(&to_string/1) - |> Enum.uniq() - ) + @spec async_fetch_blocks([%{required(:nephew_hash) => Hash.Full.t(), required(:index) => non_neg_integer()}]) :: :ok + def async_fetch_blocks(relations) when is_list(relations) do + entries = Enum.map(relations, &entry/1) + BufferedTask.buffer(__MODULE__, entries) end @doc false @@ -63,9 +59,9 @@ defmodule Indexer.Fetcher.UncleBlock do @impl BufferedTask def init(initial, reducer, _) do {:ok, final} = - Chain.stream_unfetched_uncle_hashes(initial, fn uncle_hash, acc -> - uncle_hash - |> to_string() + Chain.stream_unfetched_uncles(initial, fn uncle, acc -> + uncle + |> entry() |> reducer.(acc) end) @@ -74,31 +70,40 @@ defmodule Indexer.Fetcher.UncleBlock do @impl BufferedTask @decorate trace(name: "fetch", resource: "Indexer.Fetcher.UncleBlock.run/2", service: :indexer, tracer: Tracer) - def run(hashes, %Block.Fetcher{json_rpc_named_arguments: json_rpc_named_arguments} = block_fetcher) do - # the same block could be included as an uncle on multiple blocks, but we only want to fetch it once - unique_hashes = Enum.uniq(hashes) - - unique_hash_count = Enum.count(unique_hashes) - Logger.metadata(count: unique_hash_count) + def run(entries, %Block.Fetcher{json_rpc_named_arguments: json_rpc_named_arguments} = block_fetcher) do + entry_count = Enum.count(entries) + Logger.metadata(count: entry_count) Logger.debug("fetching") - case EthereumJSONRPC.fetch_blocks_by_hash(unique_hashes, json_rpc_named_arguments) do + entries + |> Enum.map(&entry_to_params/1) + |> EthereumJSONRPC.fetch_uncle_blocks(json_rpc_named_arguments) + |> case do {:ok, blocks} -> - run_blocks(blocks, block_fetcher, unique_hashes) + run_blocks(blocks, block_fetcher, entries) {:error, reason} -> Logger.error( fn -> ["failed to fetch: ", inspect(reason)] end, - error_count: unique_hash_count + error_count: entry_count ) - {:retry, unique_hashes} + {:retry, entries} end end + defp entry_to_params({nephew_hash_bytes, index}) when is_integer(index) do + {:ok, nephew_hash} = Hash.Full.cast(nephew_hash_bytes) + %{nephew_hash: to_string(nephew_hash), index: index} + end + + defp entry(%{nephew_hash: %Hash{bytes: nephew_hash_bytes}, index: index}) do + {nephew_hash_bytes, index} + end + defp run_blocks(%Blocks{blocks_params: []}, _, original_entries), do: {:retry, original_entries} defp run_blocks( @@ -158,9 +163,7 @@ defmodule Indexer.Fetcher.UncleBlock do # * Token.async_fetch is not called because the tokens only matter on consensus blocks # * TokenBalance.async_fetch is not called because it uses block numbers from consensus, not uncles - block_second_degree_relations - |> Enum.map(& &1.uncle_hash) - |> UncleBlock.async_fetch_blocks() + UncleBlock.async_fetch_blocks(block_second_degree_relations) ok end diff --git a/apps/indexer/lib/indexer/supervisor.ex b/apps/indexer/lib/indexer/supervisor.ex index 113aa56576..73be10f098 100644 --- a/apps/indexer/lib/indexer/supervisor.ex +++ b/apps/indexer/lib/indexer/supervisor.ex @@ -25,7 +25,8 @@ defmodule Indexer.Supervisor do alias Indexer.Temporary.{ AddressesWithoutCode, FailedCreatedAddresses, - UncatalogedTokenTransfers + UncatalogedTokenTransfers, + UnclesWithoutIndex } def child_spec([]) do @@ -129,7 +130,9 @@ defmodule Indexer.Supervisor do # Temporary workers {AddressesWithoutCode.Supervisor, [fixing_realtime_fetcher]}, {FailedCreatedAddresses.Supervisor, [json_rpc_named_arguments]}, - {UncatalogedTokenTransfers.Supervisor, [[]]} + {UncatalogedTokenTransfers.Supervisor, [[]]}, + {UnclesWithoutIndex.Supervisor, + [[json_rpc_named_arguments: json_rpc_named_arguments, memory_monitor: memory_monitor]]} ], strategy: :one_for_one ) diff --git a/apps/indexer/lib/indexer/temporary/uncles_without_index.ex b/apps/indexer/lib/indexer/temporary/uncles_without_index.ex new file mode 100644 index 0000000000..948f26b60d --- /dev/null +++ b/apps/indexer/lib/indexer/temporary/uncles_without_index.ex @@ -0,0 +1,164 @@ +defmodule Indexer.Temporary.UnclesWithoutIndex do + @moduledoc """ + Fetches `index`es for unfetched `t:Explorer.Chain.Block.SecondDegreeRelation.t/0`. + As we don't explicitly store uncle block lists for nephew blocks, we need to refetch + them completely. + """ + + use Indexer.Fetcher + use Spandex.Decorators + + require Logger + + import Ecto.Query + + alias EthereumJSONRPC.Blocks + alias Explorer.{Chain, Repo} + alias Explorer.Chain.Block.SecondDegreeRelation + alias Indexer.{BufferedTask, Tracer} + alias Indexer.Fetcher.UncleBlock + + @behaviour BufferedTask + + @defaults [ + flush_interval: :timer.seconds(3), + max_batch_size: 100, + max_concurrency: 10, + task_supervisor: Indexer.Temporary.UnclesWithoutIndex.TaskSupervisor, + metadata: [fetcher: :uncles_without_index] + ] + + @doc false + def child_spec([init_options, gen_server_options]) when is_list(init_options) do + {state, mergeable_init_options} = Keyword.pop(init_options, :json_rpc_named_arguments) + + unless state do + raise ArgumentError, + ":json_rpc_named_arguments must be provided to `#{__MODULE__}.child_spec " <> + "to allow for json_rpc calls when running." + end + + merged_init_options = + @defaults + |> Keyword.merge(mergeable_init_options) + |> Keyword.put(:state, state) + + Supervisor.child_spec({BufferedTask, [{__MODULE__, merged_init_options}, gen_server_options]}, id: __MODULE__) + end + + @impl BufferedTask + def init(initial, reducer, _) do + query = + from(bsdr in SecondDegreeRelation, + join: b in assoc(bsdr, :nephew), + where: is_nil(bsdr.index) and is_nil(bsdr.uncle_fetched_at) and b.consensus, + select: bsdr.nephew_hash, + group_by: bsdr.nephew_hash + ) + + {:ok, final} = + Repo.stream_reduce(query, initial, fn nephew_hash, acc -> + nephew_hash + |> to_string() + |> reducer.(acc) + end) + + final + end + + @impl BufferedTask + @decorate trace(name: "fetch", resource: "Indexer.Fetcher.UncleBlock.run/2", service: :indexer, tracer: Tracer) + def run(hashes, json_rpc_named_arguments) do + hash_count = Enum.count(hashes) + Logger.metadata(count: hash_count) + + Logger.debug("fetching") + + case EthereumJSONRPC.fetch_blocks_by_hash(hashes, json_rpc_named_arguments) do + {:ok, blocks} -> + run_blocks(blocks, hashes) + + {:error, reason} -> + Logger.error( + fn -> + ["failed to fetch: ", inspect(reason)] + end, + error_count: hash_count + ) + + {:retry, hashes} + end + end + + defp run_blocks(%Blocks{blocks_params: []}, original_entries), do: {:retry, original_entries} + + defp run_blocks( + %Blocks{block_second_degree_relations_params: block_second_degree_relations_params, errors: errors}, + original_entries + ) do + case Chain.import(%{block_second_degree_relations: %{params: block_second_degree_relations_params}}) do + {:ok, %{block_second_degree_relations: block_second_degree_relations}} -> + UncleBlock.async_fetch_blocks(block_second_degree_relations) + + retry(errors) + + {:error, step, failed_value, _changes_so_far} -> + Logger.error(fn -> ["failed to import: ", inspect(failed_value)] end, + step: step, + error_count: Enum.count(original_entries) + ) + + {:retry, original_entries} + end + end + + defp retry([]), do: :ok + + defp retry(errors) when is_list(errors) do + retried_entries = errors_to_entries(errors) + loggable_errors = loggable_errors(errors) + loggable_error_count = Enum.count(loggable_errors) + + unless loggable_error_count == 0 do + Logger.error( + fn -> + [ + "failed to fetch: ", + errors_to_iodata(loggable_errors) + ] + end, + error_count: loggable_error_count + ) + end + + {:retry, retried_entries} + end + + defp loggable_errors(errors) when is_list(errors) do + Enum.filter(errors, fn + %{code: 404, message: "Not Found"} -> false + _ -> true + end) + end + + defp errors_to_entries(errors) when is_list(errors) do + Enum.map(errors, &error_to_entry/1) + end + + defp error_to_entry(%{data: %{hash: hash}}) when is_binary(hash), do: hash + + defp errors_to_iodata(errors) when is_list(errors) do + errors_to_iodata(errors, []) + end + + defp errors_to_iodata([], iodata), do: iodata + + defp errors_to_iodata([error | errors], iodata) do + errors_to_iodata(errors, [iodata | error_to_iodata(error)]) + end + + defp error_to_iodata(%{code: code, message: message, data: %{hash: hash}}) + when is_integer(code) and is_binary(message) and is_binary(hash) do + [hash, ": (", to_string(code), ") ", message, ?\n] + end +end diff --git a/apps/indexer/test/indexer/block/catchup/fetcher_test.exs b/apps/indexer/test/indexer/block/catchup/fetcher_test.exs index 9588e0423d..6230552c3f 100644 --- a/apps/indexer/test/indexer/block/catchup/fetcher_test.exs +++ b/apps/indexer/test/indexer/block/catchup/fetcher_test.exs @@ -7,6 +7,7 @@ defmodule Indexer.Block.Catchup.FetcherTest do alias Explorer.Chain alias Explorer.Chain.Block.Reward + alias Explorer.Chain.Hash alias Indexer.Block alias Indexer.Block.Catchup.Fetcher alias Indexer.Fetcher.{BlockReward, CoinBalance, InternalTransaction, Token, TokenBalance, UncleBlock} @@ -51,7 +52,10 @@ defmodule Indexer.Block.Catchup.FetcherTest do Process.register(pid, UncleBlock) - nephew_hash = block_hash() |> to_string() + nephew_hash_data = block_hash() + %Hash{bytes: nephew_hash_bytes} = nephew_hash_data + nephew_hash = nephew_hash_data |> to_string() + nephew_index = 0 uncle_hash = block_hash() |> to_string() miner_hash = address_hash() |> to_string() block_number = 0 @@ -96,7 +100,8 @@ defmodule Indexer.Block.Catchup.FetcherTest do params: [ %{ nephew_hash: nephew_hash, - uncle_hash: uncle_hash + uncle_hash: uncle_hash, + index: nephew_index } ] }, @@ -113,7 +118,7 @@ defmodule Indexer.Block.Catchup.FetcherTest do } }) - assert_receive {:uncles, [^uncle_hash]} + assert_receive {:uncles, [{^nephew_hash_bytes, ^nephew_index}]} end end diff --git a/apps/indexer/test/indexer/fetcher/uncle_block_test.exs b/apps/indexer/test/indexer/fetcher/uncle_block_test.exs index 0f8bb75f02..224349d781 100644 --- a/apps/indexer/test/indexer/fetcher/uncle_block_test.exs +++ b/apps/indexer/test/indexer/fetcher/uncle_block_test.exs @@ -4,6 +4,8 @@ defmodule Indexer.Fetcher.UncleBlockTest do use EthereumJSONRPC.Case, async: false use Explorer.DataCase + import EthereumJSONRPC, only: [integer_to_quantity: 1] + alias Explorer.Chain alias Indexer.Block alias Indexer.Fetcher.UncleBlock @@ -43,16 +45,30 @@ defmodule Indexer.Fetcher.UncleBlockTest do describe "init/1" do test "fetched unfetched uncle hashes", %{json_rpc_named_arguments: json_rpc_named_arguments} do - assert %Chain.Block.SecondDegreeRelation{nephew_hash: nephew_hash, uncle_hash: uncle_hash, uncle: nil} = + assert %Chain.Block.SecondDegreeRelation{ + nephew_hash: nephew_hash, + uncle_hash: uncle_hash, + index: index, + uncle: nil + } = :block_second_degree_relation |> insert() |> Repo.preload([:nephew, :uncle]) + nephew_hash_data = to_string(nephew_hash) uncle_hash_data = to_string(uncle_hash) uncle_uncle_hash_data = to_string(block_hash()) + index_data = integer_to_quantity(index) EthereumJSONRPC.Mox - |> expect(:json_rpc, fn [%{id: id, method: "eth_getBlockByHash", params: [^uncle_hash_data, true]}], _ -> + |> expect(:json_rpc, fn [ + %{ + id: id, + method: "eth_getUncleByBlockHashAndIndex", + params: [^nephew_hash_data, ^index_data] + } + ], + _ -> number_quantity = "0x0" {:ok,