From 7d346d9975d84f36018047645f456bcea8650655 Mon Sep 17 00:00:00 2001 From: pasqu4le Date: Fri, 18 Oct 2019 15:59:33 +0200 Subject: [PATCH 01/57] Add pending block operations Problem: a lot of problems are created by tracking the importing progress on multiple tables. Querying and updating involves as such heavy operations and this is also the cause for inconsistencies. Solution: use a new table to track the progress of block's data indexing. By using a specific table, whose rows are deleted when there are no more pending operations on the relative block, we make joins quick and keep them to a minimum. This is also meant to simplify all inserts and updates and only perform deletes when strictly necessary. --- apps/explorer/lib/explorer/chain/block.ex | 4 +- .../explorer/chain/pending_block_operation.ex | 46 +++++++++++++++++++ ...120546_create_pending_block_operations.exs | 14 ++++++ 3 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 apps/explorer/lib/explorer/chain/pending_block_operation.ex create mode 100644 apps/explorer/priv/repo/migrations/20191018120546_create_pending_block_operations.exs diff --git a/apps/explorer/lib/explorer/chain/block.ex b/apps/explorer/lib/explorer/chain/block.ex index 32905cf076..0035eb2688 100644 --- a/apps/explorer/lib/explorer/chain/block.ex +++ b/apps/explorer/lib/explorer/chain/block.ex @@ -7,7 +7,7 @@ defmodule Explorer.Chain.Block do use Explorer.Schema - alias Explorer.Chain.{Address, Gas, Hash, Transaction} + alias Explorer.Chain.{Address, Gas, Hash, PendingBlockOperation, Transaction} alias Explorer.Chain.Block.{Reward, SecondDegreeRelation} @optional_attrs ~w(internal_transactions_indexed_at size refetch_needed total_difficulty difficulty)a @@ -97,6 +97,8 @@ defmodule Explorer.Chain.Block do has_many(:transaction_forks, Transaction.Fork, foreign_key: :uncle_hash) has_many(:rewards, Reward, foreign_key: :block_hash) + + has_one(:pending_operations, PendingBlockOperation, foreign_key: :block_hash) end def changeset(%__MODULE__{} = block, attrs) do diff --git a/apps/explorer/lib/explorer/chain/pending_block_operation.ex b/apps/explorer/lib/explorer/chain/pending_block_operation.ex new file mode 100644 index 0000000000..fa6ad4c2a5 --- /dev/null +++ b/apps/explorer/lib/explorer/chain/pending_block_operation.ex @@ -0,0 +1,46 @@ +defmodule Explorer.Chain.PendingBlockOperation do + @moduledoc """ + Tracks a block that has pending operations. + """ + + use Explorer.Schema + + alias Explorer.Chain.{Block, Hash} + + @required_attrs ~w(block_hash)a + + @typedoc """ + * `block_hash` - the hash of the block that has pending operations. + """ + @type t :: %__MODULE__{ + block_hash: Hash.Full.t() + } + + @primary_key false + schema "pending_block_operations" do + timestamps() + + belongs_to(:block, Block, foreign_key: :block_hash, primary_key: true, references: :hash, type: Hash.Full) + end + + def changeset(%__MODULE__{} = pending_ops, attrs) do + pending_ops + |> cast(attrs, @required_attrs) + |> validate_required(@required_attrs) + |> foreign_key_constraint(:block_hash) + |> unique_constraint(:block_hash, name: :pending_block_operations_pkey) + end + + @doc """ + Returns all pending block operations with the `block_hash` in the given list, + using "FOR UPDATE" to grab ShareLocks in order (see docs: sharelocks.md) + """ + def fetch_and_lock_by_hashes(hashes) when is_list(hashes) do + from( + pending_ops in __MODULE__, + where: pending_ops.block_hash in ^hashes, + order_by: [asc: pending_ops.block_hash], + lock: "FOR UPDATE" + ) + end +end diff --git a/apps/explorer/priv/repo/migrations/20191018120546_create_pending_block_operations.exs b/apps/explorer/priv/repo/migrations/20191018120546_create_pending_block_operations.exs new file mode 100644 index 0000000000..3e917122a6 --- /dev/null +++ b/apps/explorer/priv/repo/migrations/20191018120546_create_pending_block_operations.exs @@ -0,0 +1,14 @@ +defmodule Explorer.Repo.Migrations.CreatePendingBlockOperations do + use Ecto.Migration + + def change do + create table(:pending_block_operations, primary_key: false) do + add(:block_hash, references(:blocks, column: :hash, type: :bytea, on_delete: :delete_all), + null: false, + primary_key: true + ) + + timestamps(null: false, type: :utc_datetime_usec) + end + end +end From 99cb7972aebe79a61912d2dc383cb373c05b4243 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Fri, 22 Nov 2019 10:01:37 +0300 Subject: [PATCH 02/57] add block_hash to token_transfers --- .../explorer/chain/import/runner/blocks.ex | 34 +------------------ .../chain/import/runner/token_transfers.ex | 2 +- .../lib/explorer/chain/token_transfer.ex | 11 +++++- ...2035_add_block_hash_to_token_transfers.exs | 33 ++++++++++++++++++ .../lib/indexer/transform/token_transfers.ex | 3 ++ 5 files changed, 48 insertions(+), 35 deletions(-) create mode 100644 apps/explorer/priv/repo/migrations/20191122062035_add_block_hash_to_token_transfers.exs diff --git a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex index 61fb6973c9..2a47892e3f 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex @@ -8,7 +8,7 @@ defmodule Explorer.Chain.Import.Runner.Blocks do import Ecto.Query, only: [from: 2, subquery: 1] alias Ecto.{Changeset, Multi, Repo} - alias Explorer.Chain.{Address, Block, Import, InternalTransaction, Log, TokenTransfer, Transaction} + alias Explorer.Chain.{Address, Block, Import, InternalTransaction, Log, Transaction} alias Explorer.Chain.Block.Reward alias Explorer.Chain.Import.Runner alias Explorer.Chain.Import.Runner.Address.CurrentTokenBalances @@ -92,9 +92,6 @@ defmodule Explorer.Chain.Import.Runner.Blocks do |> Multi.run(:acquire_contract_address_tokens, fn repo, _ -> acquire_contract_address_tokens(repo, consensus_block_numbers) end) - |> Multi.run(:remove_nonconsensus_token_transfers, fn repo, %{derive_transaction_forks: transactions} -> - remove_nonconsensus_token_transfers(repo, transactions, insert_options) - end) |> Multi.run(:delete_address_token_balances, fn repo, _ -> delete_address_token_balances(repo, consensus_block_numbers, insert_options) end) @@ -317,35 +314,6 @@ defmodule Explorer.Chain.Import.Runner.Blocks do {:error, %{exception: postgrex_error, consensus_block_numbers: consensus_block_numbers}} end - defp remove_nonconsensus_token_transfers(repo, forked_transaction_hashes, %{timeout: timeout}) do - ordered_token_transfers = - from( - token_transfer in TokenTransfer, - where: token_transfer.transaction_hash in ^forked_transaction_hashes, - select: token_transfer.transaction_hash, - # Enforce TokenTransfer ShareLocks order (see docs: sharelocks.md) - order_by: [ - token_transfer.transaction_hash, - token_transfer.log_index - ], - lock: "FOR UPDATE" - ) - - query = - from(token_transfer in TokenTransfer, - select: map(token_transfer, [:transaction_hash, :log_index]), - inner_join: ordered_token_transfer in subquery(ordered_token_transfers), - on: ordered_token_transfer.transaction_hash == token_transfer.transaction_hash - ) - - {_count, deleted_token_transfers} = repo.delete_all(query, timeout: timeout) - - {:ok, deleted_token_transfers} - rescue - postgrex_error in Postgrex.Error -> - {:error, %{exception: postgrex_error, transactions: forked_transaction_hashes}} - end - defp remove_nonconsensus_internal_transactions(repo, forked_transaction_hashes, %{timeout: timeout}) do query = from( diff --git a/apps/explorer/lib/explorer/chain/import/runner/token_transfers.ex b/apps/explorer/lib/explorer/chain/import/runner/token_transfers.ex index cc88389f4c..28b1f5ebd3 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/token_transfers.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/token_transfers.ex @@ -61,7 +61,7 @@ defmodule Explorer.Chain.Import.Runner.TokenTransfers do Import.insert_changes_list( repo, ordered_changes_list, - conflict_target: [:transaction_hash, :log_index], + conflict_target: [:transaction_hash, :log_index, :block_hash], on_conflict: on_conflict, for: TokenTransfer, returning: true, diff --git a/apps/explorer/lib/explorer/chain/token_transfer.ex b/apps/explorer/lib/explorer/chain/token_transfer.ex index b6171219bf..5eced56741 100644 --- a/apps/explorer/lib/explorer/chain/token_transfer.ex +++ b/apps/explorer/lib/explorer/chain/token_transfer.ex @@ -27,7 +27,7 @@ defmodule Explorer.Chain.TokenTransfer do import Ecto.Changeset import Ecto.Query, only: [from: 2, limit: 2, where: 3] - alias Explorer.Chain.{Address, Hash, TokenTransfer, Transaction} + alias Explorer.Chain.{Address, Block, Hash, TokenTransfer, Transaction} alias Explorer.Chain.Token.Instance alias Explorer.{PagingOptions, Repo} @@ -35,6 +35,7 @@ defmodule Explorer.Chain.TokenTransfer do @typedoc """ * `:amount` - The token transferred amount + * `:block_hash` - hash of the block * `:block_number` - The block number that the transfer took place. * `:from_address` - The `t:Explorer.Chain.Address.t/0` that sent the tokens * `:from_address_hash` - Address hash foreign key @@ -50,6 +51,7 @@ defmodule Explorer.Chain.TokenTransfer do @type t :: %TokenTransfer{ amount: Decimal.t(), block_number: non_neg_integer() | nil, + block_hash: Hash.Full.t(), from_address: %Ecto.Association.NotLoaded{} | Address.t(), from_address_hash: Hash.Address.t(), to_address: %Ecto.Association.NotLoaded{} | Address.t(), @@ -93,6 +95,13 @@ defmodule Explorer.Chain.TokenTransfer do type: Hash.Full ) + belongs_to(:block, Block, + foreign_key: :block_hash, + primary_key: true, + references: :hash, + type: Hash.Full + ) + has_one( :instance, Instance, diff --git a/apps/explorer/priv/repo/migrations/20191122062035_add_block_hash_to_token_transfers.exs b/apps/explorer/priv/repo/migrations/20191122062035_add_block_hash_to_token_transfers.exs new file mode 100644 index 0000000000..c3ee02f605 --- /dev/null +++ b/apps/explorer/priv/repo/migrations/20191122062035_add_block_hash_to_token_transfers.exs @@ -0,0 +1,33 @@ +defmodule Explorer.Repo.Migrations.AddBlockHashToTokenTransfers do + use Ecto.Migration + + def change do + alter table(:token_transfers) do + add(:block_hash, :bytea) + end + + execute(""" + UPDATE token_transfers token_transfer + SET block_hash = with_block.block_hash + FROM ( + SELECT transfer.transaction_hash, + t.block_hash + FROM token_transfers transfer + JOIN transactions t + ON t.hash = transfer.transaction_hash + ) AS with_block + WHERE token_transfer.transaction_hash = with_block.transaction_hash + ; + """) + + alter table(:token_transfers) do + modify(:block_hash, references(:blocks, column: :hash, type: :bytea), null: false) + end + + execute(""" + ALTER table token_transfers + DROP CONSTRAINT token_transfers_pkey, + ADD PRIMARY KEY (transaction_hash, block_hash, log_index); + """) + end +end diff --git a/apps/indexer/lib/indexer/transform/token_transfers.ex b/apps/indexer/lib/indexer/transform/token_transfers.ex index 58ea1e57ef..303efa059e 100644 --- a/apps/indexer/lib/indexer/transform/token_transfers.ex +++ b/apps/indexer/lib/indexer/transform/token_transfers.ex @@ -40,6 +40,7 @@ defmodule Indexer.Transform.TokenTransfers do token_transfer = %{ amount: Decimal.new(amount || 0), block_number: log.block_number, + block_hash: log.block_hash, log_index: log.index, from_address_hash: truncate_address_hash(log.second_topic), to_address_hash: truncate_address_hash(log.third_topic), @@ -64,6 +65,7 @@ defmodule Indexer.Transform.TokenTransfers do token_transfer = %{ block_number: log.block_number, log_index: log.index, + block_hash: log.block_hash, from_address_hash: truncate_address_hash(log.second_topic), to_address_hash: truncate_address_hash(log.third_topic), token_contract_address_hash: log.address_hash, @@ -87,6 +89,7 @@ defmodule Indexer.Transform.TokenTransfers do token_transfer = %{ block_number: log.block_number, + block_hash: log.block_hash, log_index: log.index, from_address_hash: encode_address_hash(from_address_hash), to_address_hash: encode_address_hash(to_address_hash), From 2aac0929604b809074b986feb26d71ed56afa818 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Sat, 23 Nov 2019 12:56:39 +0300 Subject: [PATCH 03/57] parse block hash from logs --- apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/log.ex | 2 ++ .../lib/explorer/chain/import/runner/token_transfers.ex | 2 +- apps/explorer/lib/explorer/chain/token_transfer.ex | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/log.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/log.ex index c9a6fd08e6..1ebc1b476c 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/log.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/log.ex @@ -84,6 +84,7 @@ defmodule EthereumJSONRPC.Log do %{ "address" => address_hash, "blockNumber" => block_number, + "blockHash" => block_hash, "data" => data, "logIndex" => index, "topics" => topics, @@ -93,6 +94,7 @@ defmodule EthereumJSONRPC.Log do %{ address_hash: address_hash, block_number: block_number, + block_hash: block_hash, data: data, index: index, transaction_hash: transaction_hash diff --git a/apps/explorer/lib/explorer/chain/import/runner/token_transfers.ex b/apps/explorer/lib/explorer/chain/import/runner/token_transfers.ex index 28b1f5ebd3..3935fb53b5 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/token_transfers.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/token_transfers.ex @@ -55,7 +55,7 @@ defmodule Explorer.Chain.Import.Runner.TokenTransfers do on_conflict = Map.get_lazy(options, :on_conflict, &default_on_conflict/0) # Enforce TokenTransfer ShareLocks order (see docs: sharelocks.md) - ordered_changes_list = Enum.sort_by(changes_list, &{&1.transaction_hash, &1.log_index}) + ordered_changes_list = Enum.sort_by(changes_list, &{&1.transaction_hash, &1.block_hash, &1.log_index}) {:ok, _} = Import.insert_changes_list( diff --git a/apps/explorer/lib/explorer/chain/token_transfer.ex b/apps/explorer/lib/explorer/chain/token_transfer.ex index 5eced56741..22e59b23c7 100644 --- a/apps/explorer/lib/explorer/chain/token_transfer.ex +++ b/apps/explorer/lib/explorer/chain/token_transfer.ex @@ -114,7 +114,7 @@ defmodule Explorer.Chain.TokenTransfer do timestamps() end - @required_attrs ~w(block_number log_index from_address_hash to_address_hash token_contract_address_hash transaction_hash)a + @required_attrs ~w(block_number log_index from_address_hash to_address_hash token_contract_address_hash transaction_hash block_hash)a @optional_attrs ~w(amount token_id)a @doc false From 673fc0fcdfc54bc4c5d967f4a61de7b671feb80c Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Sat, 23 Nov 2019 13:06:19 +0300 Subject: [PATCH 04/57] fix token transfer factory --- apps/explorer/test/explorer/chain_test.exs | 1 + apps/explorer/test/support/factory.ex | 1 + 2 files changed, 2 insertions(+) diff --git a/apps/explorer/test/explorer/chain_test.exs b/apps/explorer/test/explorer/chain_test.exs index 5aa8a6b1d2..5b776a99e8 100644 --- a/apps/explorer/test/explorer/chain_test.exs +++ b/apps/explorer/test/explorer/chain_test.exs @@ -1343,6 +1343,7 @@ defmodule Explorer.ChainTest do token_transfers: %{ params: [ %{ + block_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd", amount: Decimal.new(1_000_000_000_000_000_000), block_number: 37, log_index: 0, diff --git a/apps/explorer/test/support/factory.ex b/apps/explorer/test/support/factory.ex index b2a1feeac6..1d560630df 100644 --- a/apps/explorer/test/support/factory.ex +++ b/apps/explorer/test/support/factory.ex @@ -417,6 +417,7 @@ defmodule Explorer.Factory do insert(:token, contract_address: token_address) %TokenTransfer{ + block: build(:block), amount: Decimal.new(1), block_number: block_number(), from_address: from_address, From d37fb1d579d526062641cfafc81a1cf0c6545bd5 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Sat, 23 Nov 2019 13:28:20 +0300 Subject: [PATCH 05/57] fix most tests --- .../controllers/api/rpc/address_controller_test.exs | 2 +- apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/log.ex | 2 ++ .../test/ethereum_jsonrpc/receipts_test.exs | 3 +++ .../test/indexer/transform/token_transfers_test.exs | 12 ++++++++++-- 4 files changed, 16 insertions(+), 3 deletions(-) diff --git a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/address_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/address_controller_test.exs index 135fbe9948..ba9725bb1c 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/address_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/address_controller_test.exs @@ -1817,7 +1817,7 @@ defmodule BlockScoutWeb.API.RPC.AddressControllerTest do |> insert() |> with_block() - token_transfer = insert(:token_transfer, transaction: transaction) + token_transfer = insert(:token_transfer, block: transaction.block, transaction: transaction) {:ok, token} = Chain.token_from_address_hash(token_transfer.token_contract_address_hash) params = %{ diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/log.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/log.ex index 1ebc1b476c..52cdae96bd 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/log.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/log.ex @@ -39,6 +39,7 @@ defmodule EthereumJSONRPC.Log do ...> ) %{ address_hash: "0x8bf38d4764929064f2d4d3a56520a76ab3df415b", + block_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd", block_number: 37, data: "0x000000000000000000000000862d67cb0773ee3f8ce7ea89b328ffea861ab3ef", first_topic: "0x600bcf04a13e752d1e3670a5a9f1c21177ca2a93c6f5391d4f1298d098097c22", @@ -69,6 +70,7 @@ defmodule EthereumJSONRPC.Log do ...> ) %{ address_hash: "0xda8b3276cde6d768a44b9dac659faa339a41ac55", + block_hash: "0x0b89f7f894f5d8ba941e16b61490e999a0fcaaf92dfcc70aee2ac5ddb5f243e1", block_number: 4448, data: "0x290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563", first_topic: "0xadc1e8a294f8415511303acc4a8c0c5906c7eb0bf2a71043d7f4b03b46a39130", diff --git a/apps/ethereum_jsonrpc/test/ethereum_jsonrpc/receipts_test.exs b/apps/ethereum_jsonrpc/test/ethereum_jsonrpc/receipts_test.exs index d8eef3b6da..468e0f0f30 100644 --- a/apps/ethereum_jsonrpc/test/ethereum_jsonrpc/receipts_test.exs +++ b/apps/ethereum_jsonrpc/test/ethereum_jsonrpc/receipts_test.exs @@ -34,6 +34,7 @@ defmodule EthereumJSONRPC.ReceiptsTest do cumulative_gas_used: 884_322, address_hash: "0x1e2fbe6be9eb39fc894d38be976111f332172d83", block_number: 3_560_000, + block_hash: nil, data: "0x00000000000000000000000033066f6a8adf2d4f5db193524b6fbae062ec0d110000000000000000000000000000000000000000000000000000000000001030", index: 12, @@ -48,6 +49,7 @@ defmodule EthereumJSONRPC.ReceiptsTest do EthereumJSONRPC.Parity -> %{ created_contract_address_hash: nil, + block_hash: nil, cumulative_gas_used: 50450, gas_used: 50450, address_hash: "0x8bf38d4764929064f2d4d3a56520a76ab3df415b", @@ -83,6 +85,7 @@ defmodule EthereumJSONRPC.ReceiptsTest do %{ "address" => address_hash, "blockNumber" => integer_to_quantity(block_number), + "blockHash" => nil, "data" => data, "logIndex" => integer_to_quantity(index), "topics" => [first_topic], diff --git a/apps/indexer/test/indexer/transform/token_transfers_test.exs b/apps/indexer/test/indexer/transform/token_transfers_test.exs index 406107b187..b38c1d4088 100644 --- a/apps/indexer/test/indexer/transform/token_transfers_test.exs +++ b/apps/indexer/test/indexer/transform/token_transfers_test.exs @@ -12,6 +12,7 @@ defmodule Indexer.Transform.TokenTransfersTest do %{ address_hash: "0xf2eec76e45b328df99a34fa696320a262cb92154", block_number: 3_530_917, + block_hash: "0x79594150677f083756a37eee7b97ed99ab071f502104332cb3835bac345711ca", data: "0x000000000000000000000000000000000000000000000000ebec21ee1da40000", first_topic: "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", fourth_topic: nil, @@ -23,6 +24,7 @@ defmodule Indexer.Transform.TokenTransfersTest do }, %{ address_hash: "0x6ea5ec9cb832e60b6b1654f5826e9be638f276a5", + block_hash: "0x79594150677f083756a37eee7b97ed99ab071f502104332cb3835bac345711ca", block_number: 3_586_935, data: "0x", first_topic: "0x55e10366a5f552746106978b694d7ef3bbddec06bd5f9b9d15ad46f475c653ef", @@ -36,6 +38,7 @@ defmodule Indexer.Transform.TokenTransfersTest do %{ address_hash: "0x91932e8c6776fb2b04abb71874a7988747728bb2", block_number: 3_664_064, + block_hash: "0x79594150677f083756a37eee7b97ed99ab071f502104332cb3835bac345711ca", data: "0x000000000000000000000000000000000000000000000000ebec21ee1da40000", first_topic: "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", fourth_topic: "0x00000000000000000000000000000000000000000000000000000000000000b7", @@ -67,7 +70,8 @@ defmodule Indexer.Transform.TokenTransfersTest do token_contract_address_hash: log_3.address_hash, token_id: 183, transaction_hash: log_3.transaction_hash, - token_type: "ERC-721" + token_type: "ERC-721", + block_hash: log_3.block_hash }, %{ amount: Decimal.new(17_000_000_000_000_000_000), @@ -77,7 +81,8 @@ defmodule Indexer.Transform.TokenTransfersTest do to_address_hash: truncated_hash(log_1.third_topic), token_contract_address_hash: log_1.address_hash, transaction_hash: log_1.transaction_hash, - token_type: "ERC-20" + token_type: "ERC-20", + block_hash: log_1.block_hash } ] } @@ -97,6 +102,7 @@ defmodule Indexer.Transform.TokenTransfersTest do second_topic: nil, third_topic: nil, transaction_hash: "0x6d2dd62c178e55a13b65601f227c4ffdd8aa4e3bcb1f24731363b4f7619e92c8", + block_hash: "0x79594150677f083756a37eee7b97ed99ab071f502104332cb3835bac345711ca", type: "mined" } @@ -113,6 +119,7 @@ defmodule Indexer.Transform.TokenTransfersTest do log_index: log.index, from_address_hash: "0x58ab73cb79c8275628e0213742a85b163fe0a9fb", to_address_hash: "0xbe8cdfc13ffda20c844ac3da2b53a23ac5787f1e", + block_hash: "0x79594150677f083756a37eee7b97ed99ab071f502104332cb3835bac345711ca", token_contract_address_hash: log.address_hash, token_id: 14_939, transaction_hash: log.transaction_hash, @@ -128,6 +135,7 @@ defmodule Indexer.Transform.TokenTransfersTest do log = %{ address_hash: "0x58Ab73CB79c8275628E0213742a85B163fE0A9Fb", block_number: 8_683_457, + block_hash: "0x79594150677f083756a37eee7b97ed99ab071f502104332cb3835bac345711ca", data: "0x", first_topic: "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", fourth_topic: nil, From f17ba4db8c908afa0e8c62fa02c45f0bc0d9d40a Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Sat, 23 Nov 2019 13:45:28 +0300 Subject: [PATCH 06/57] fix remaining tests --- .../chain/import/runner/blocks_test.exs | 38 ------------------- .../test/explorer/chain/import_test.exs | 2 + 2 files changed, 2 insertions(+), 38 deletions(-) diff --git a/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs b/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs index c47204e749..5eaaca5344 100644 --- a/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs +++ b/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs @@ -115,44 +115,6 @@ defmodule Explorer.Chain.Import.Runner.BlocksTest do assert count(Address.CurrentTokenBalance) == count end - test "remove_nonconsensus_token_transfers deletes token transfer rows with matching block number when new consensus block is inserted", - %{consensus_block: %{number: block_number} = block, options: options} do - consensus_block = insert(:block, number: block_number, consensus: true) - - transaction = insert(:transaction) |> with_block(consensus_block) - - %TokenTransfer{transaction_hash: transaction_hash, log_index: log_index} = - insert(:token_transfer, block_number: block_number, transaction: transaction) - - assert count(TokenTransfer) == 1 - - assert {:ok, - %{ - remove_nonconsensus_token_transfers: [ - %{transaction_hash: ^transaction_hash, log_index: ^log_index} - ] - }} = run_block_consensus_change(block, true, options) - - assert count(TokenTransfer) == 0 - end - - test "remove_nonconsensus_token_transfers does not delete token transfer rows with matching block number when new consensus block wasn't inserted", - %{consensus_block: %{number: block_number} = block, options: options} do - consensus_block = insert(:block, number: block_number, consensus: true) - - transaction = insert(:transaction) |> with_block(consensus_block) - - insert(:token_transfer, block_number: block_number, transaction: transaction) - - count = 1 - - assert count(TokenTransfer) == count - - assert {:ok, %{remove_nonconsensus_token_transfers: []}} = run_block_consensus_change(block, false, options) - - assert count(TokenTransfer) == count - end - test "remove_nonconsensus_logs deletes nonconsensus logs", %{ consensus_block: %{number: block_number} = block, options: options diff --git a/apps/explorer/test/explorer/chain/import_test.exs b/apps/explorer/test/explorer/chain/import_test.exs index 0c559bdfec..4a2fe41c65 100644 --- a/apps/explorer/test/explorer/chain/import_test.exs +++ b/apps/explorer/test/explorer/chain/import_test.exs @@ -149,6 +149,7 @@ defmodule Explorer.Chain.ImportTest do %{ amount: Decimal.new(1_000_000_000_000_000_000), block_number: 37, + block_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd", log_index: 0, from_address_hash: "0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca", to_address_hash: "0x515c09c5bba1ed566b02a5b0599ec5d5d0aee73d", @@ -1571,6 +1572,7 @@ defmodule Explorer.Chain.ImportTest do params: [ params_for( :token_transfer, + block_hash: block_hash, block_number: 35, from_address_hash: from_address_hash, to_address_hash: to_address_hash, From 05218c6a1356822d1aef6a38f97bfa6a9ac9458b Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Sat, 23 Nov 2019 13:52:37 +0300 Subject: [PATCH 07/57] add CHANGELOG entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 119599b845..bf01641a5f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ - [#2830](https://github.com/poanetwork/blockscout/pull/2830) - Fix wrong color of contract icon on xDai chain - [#2829](https://github.com/poanetwork/blockscout/pull/2829) - Fix for stuck gas limit label and value - [#2828](https://github.com/poanetwork/blockscout/pull/2828) - Fix for script that clears compilation/launching assets +- [#2872](https://github.com/poanetwork/blockscout/pull/2872) - do not remove token transfers - [#2800](https://github.com/poanetwork/blockscout/pull/2800) - return not found for not verified contract for token read_contract - [#2806](https://github.com/poanetwork/blockscout/pull/2806) - Fix blocks fetching on the main page - [#2803](https://github.com/poanetwork/blockscout/pull/2803) - Fix block validator custom tooltip From bd5a9e72564383ac118d4d62a4edd665ce766f73 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Sun, 24 Nov 2019 09:08:21 +0300 Subject: [PATCH 08/57] fix query --- apps/explorer/lib/explorer/chain.ex | 5 ++++- apps/explorer/test/explorer/chain_test.exs | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index 9576eca1a6..614a6cff4f 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -2495,7 +2495,10 @@ defmodule Explorer.Chain do TokenTransfer |> join(:inner, [token_transfer], transaction in assoc(token_transfer, :transaction)) - |> where([_, transaction], transaction.hash == ^transaction_hash) + |> where( + [token_transfer, transaction], + transaction.hash == ^transaction_hash and token_transfer.block_hash == transaction.block_hash + ) |> TokenTransfer.page_token_transfer(paging_options) |> limit(^paging_options.page_size) |> order_by([token_transfer], asc: token_transfer.inserted_at) diff --git a/apps/explorer/test/explorer/chain_test.exs b/apps/explorer/test/explorer/chain_test.exs index 5b776a99e8..abb0f9a360 100644 --- a/apps/explorer/test/explorer/chain_test.exs +++ b/apps/explorer/test/explorer/chain_test.exs @@ -2501,7 +2501,7 @@ defmodule Explorer.ChainTest do |> with_block() %TokenTransfer{transaction_hash: transaction_hash, log_index: log_index} = - insert(:token_transfer, transaction: transaction) + insert(:token_transfer, transaction: transaction, block: transaction.block) assert [%TokenTransfer{transaction_hash: ^transaction_hash, log_index: ^log_index}] = Chain.transaction_to_token_transfers(transaction.hash) @@ -2513,7 +2513,7 @@ defmodule Explorer.ChainTest do |> insert() |> with_block() - insert(:token_transfer, transaction: transaction) + insert(:token_transfer, transaction: transaction, block: transaction.block) assert [%TokenTransfer{token: %Token{}, transaction: %Transaction{}}] = Chain.transaction_to_token_transfers( From f3e9b58fedeb0b43d23283b198d404e01297f3d5 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Sun, 24 Nov 2019 09:17:05 +0300 Subject: [PATCH 09/57] fix tests --- .../transaction_token_transfer_controller_test.exs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/apps/block_scout_web/test/block_scout_web/controllers/transaction_token_transfer_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/transaction_token_transfer_controller_test.exs index 6d3c83980b..72da1bc63d 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/transaction_token_transfer_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/transaction_token_transfer_controller_test.exs @@ -46,11 +46,11 @@ defmodule BlockScoutWeb.TransactionTokenTransferControllerTest do end test "includes token transfers for the transaction", %{conn: conn} do - transaction = insert(:transaction) + transaction = insert(:transaction) |> with_block() - insert(:token_transfer, transaction: transaction) + insert(:token_transfer, transaction: transaction, block: transaction.block) - insert(:token_transfer, transaction: transaction) + insert(:token_transfer, transaction: transaction, block: transaction.block) path = transaction_token_transfer_path(BlockScoutWeb.Endpoint, :index, transaction.hash) @@ -106,7 +106,8 @@ defmodule BlockScoutWeb.TransactionTokenTransferControllerTest do insert( :token_transfer, transaction: transaction, - log_index: log_index + log_index: log_index, + block: transaction.block ) end) From fa1ace24e2c6a53df6d8d5eb8194573d5e9b8eb2 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 26 Nov 2019 14:45:22 +0300 Subject: [PATCH 10/57] select by block_number --- apps/explorer/lib/explorer/chain.ex | 3 ++- apps/explorer/test/explorer/chain_test.exs | 8 ++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index 614a6cff4f..1475c97c99 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -2497,7 +2497,8 @@ defmodule Explorer.Chain do |> join(:inner, [token_transfer], transaction in assoc(token_transfer, :transaction)) |> where( [token_transfer, transaction], - transaction.hash == ^transaction_hash and token_transfer.block_hash == transaction.block_hash + transaction.hash == ^transaction_hash and token_transfer.block_hash == transaction.block_hash and + token_transfer.block_number == transaction.block_number ) |> TokenTransfer.page_token_transfer(paging_options) |> limit(^paging_options.page_size) diff --git a/apps/explorer/test/explorer/chain_test.exs b/apps/explorer/test/explorer/chain_test.exs index abb0f9a360..148897925f 100644 --- a/apps/explorer/test/explorer/chain_test.exs +++ b/apps/explorer/test/explorer/chain_test.exs @@ -2501,7 +2501,11 @@ defmodule Explorer.ChainTest do |> with_block() %TokenTransfer{transaction_hash: transaction_hash, log_index: log_index} = - insert(:token_transfer, transaction: transaction, block: transaction.block) + insert(:token_transfer, + transaction: transaction, + block: transaction.block, + block_number: transaction.block_number + ) assert [%TokenTransfer{transaction_hash: ^transaction_hash, log_index: ^log_index}] = Chain.transaction_to_token_transfers(transaction.hash) @@ -2513,7 +2517,7 @@ defmodule Explorer.ChainTest do |> insert() |> with_block() - insert(:token_transfer, transaction: transaction, block: transaction.block) + insert(:token_transfer, transaction: transaction, block: transaction.block, block_number: transaction.block_number) assert [%TokenTransfer{token: %Token{}, transaction: %Transaction{}}] = Chain.transaction_to_token_transfers( From e9c56a5d571794e7cee23ee58ca501dc9b30f55e Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 26 Nov 2019 15:58:36 +0300 Subject: [PATCH 11/57] select based on block_number and block_hash in token_transfers rpc --- apps/explorer/lib/explorer/etherscan.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/explorer/lib/explorer/etherscan.ex b/apps/explorer/lib/explorer/etherscan.ex index 71ee5794f7..90b3b96324 100644 --- a/apps/explorer/lib/explorer/etherscan.ex +++ b/apps/explorer/lib/explorer/etherscan.ex @@ -321,7 +321,7 @@ defmodule Explorer.Etherscan do inner_join: tt in assoc(t, :token_transfers), inner_join: tkn in assoc(tt, :token), inner_join: b in assoc(t, :block), - where: tt.from_address_hash == ^address_hash, + where: tt.from_address_hash == ^address_hash and b.number == tt.block_number and b.hash == t.block_hash, or_where: tt.to_address_hash == ^address_hash, order_by: [{^options.order_by_direction, t.block_number}], limit: ^options.page_size, From 8aa55932f704c003f333cbf333a2c7eeb0d34f77 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 26 Nov 2019 16:42:17 +0300 Subject: [PATCH 12/57] fix tests --- .../api/rpc/address_controller_test.exs | 24 ++++- ...saction_token_transfer_controller_test.exs | 11 ++- .../explorer/test/explorer/etherscan_test.exs | 98 ++++++++++++++++--- 3 files changed, 112 insertions(+), 21 deletions(-) diff --git a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/address_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/address_controller_test.exs index ba9725bb1c..2215057a9a 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/address_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/address_controller_test.exs @@ -1787,7 +1787,9 @@ defmodule BlockScoutWeb.API.RPC.AddressControllerTest do insert(:token_transfer, %{ token_contract_address: token_address, token_id: 666, - transaction: transaction + transaction: transaction, + block: transaction.block, + block_number: transaction.block_number }) {:ok, _} = Chain.token_from_address_hash(token_transfer.token_contract_address_hash) @@ -1817,7 +1819,9 @@ defmodule BlockScoutWeb.API.RPC.AddressControllerTest do |> insert() |> with_block() - token_transfer = insert(:token_transfer, block: transaction.block, transaction: transaction) + token_transfer = + insert(:token_transfer, block: transaction.block, transaction: transaction, block_number: block.number) + {:ok, token} = Chain.token_from_address_hash(token_transfer.token_contract_address_hash) params = %{ @@ -1894,8 +1898,20 @@ defmodule BlockScoutWeb.API.RPC.AddressControllerTest do |> insert() |> with_block() - insert(:token_transfer, from_address: address, transaction: transaction) - insert(:token_transfer, from_address: address, token_contract_address: contract_address, transaction: transaction) + insert(:token_transfer, + from_address: address, + transaction: transaction, + block: transaction.block, + block_number: transaction.block_number + ) + + insert(:token_transfer, + from_address: address, + token_contract_address: contract_address, + transaction: transaction, + block: transaction.block, + block_number: transaction.block_number + ) params = %{ "module" => "account", diff --git a/apps/block_scout_web/test/block_scout_web/controllers/transaction_token_transfer_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/transaction_token_transfer_controller_test.exs index 72da1bc63d..3c1fb0a8f4 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/transaction_token_transfer_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/transaction_token_transfer_controller_test.exs @@ -48,9 +48,9 @@ defmodule BlockScoutWeb.TransactionTokenTransferControllerTest do test "includes token transfers for the transaction", %{conn: conn} do transaction = insert(:transaction) |> with_block() - insert(:token_transfer, transaction: transaction, block: transaction.block) + insert(:token_transfer, transaction: transaction, block: transaction.block, block_number: transaction.block_number) - insert(:token_transfer, transaction: transaction, block: transaction.block) + insert(:token_transfer, transaction: transaction, block: transaction.block, block_number: transaction.block_number) path = transaction_token_transfer_path(BlockScoutWeb.Endpoint, :index, transaction.hash) @@ -107,7 +107,8 @@ defmodule BlockScoutWeb.TransactionTokenTransferControllerTest do :token_transfer, transaction: transaction, log_index: log_index, - block: transaction.block + block: transaction.block, + block_number: transaction.block_number ) end) @@ -130,7 +131,9 @@ defmodule BlockScoutWeb.TransactionTokenTransferControllerTest do insert( :token_transfer, transaction: transaction, - log_index: log_index + log_index: log_index, + block_number: transaction.block_number, + block: transaction.block ) end) diff --git a/apps/explorer/test/explorer/etherscan_test.exs b/apps/explorer/test/explorer/etherscan_test.exs index 5d1cf7fcb1..5ba4b8dc8e 100644 --- a/apps/explorer/test/explorer/etherscan_test.exs +++ b/apps/explorer/test/explorer/etherscan_test.exs @@ -828,7 +828,12 @@ defmodule Explorer.EtherscanTest do |> insert() |> with_block() - token_transfer = insert(:token_transfer, transaction: transaction) + token_transfer = + insert(:token_transfer, + transaction: transaction, + block: transaction.block, + block_number: transaction.block_number + ) [found_token_transfer] = Etherscan.list_token_transfers(token_transfer.from_address_hash, nil) @@ -863,9 +868,26 @@ defmodule Explorer.EtherscanTest do |> insert() |> with_block() - insert(:token_transfer, from_address: address1, transaction: transaction) - insert(:token_transfer, from_address: address1, transaction: transaction) - insert(:token_transfer, from_address: address2, transaction: transaction) + insert(:token_transfer, + from_address: address1, + transaction: transaction, + block: transaction.block, + block_number: transaction.block_number + ) + + insert(:token_transfer, + from_address: address1, + transaction: transaction, + block: transaction.block, + block_number: transaction.block_number + ) + + insert(:token_transfer, + from_address: address2, + transaction: transaction, + block: transaction.block, + block_number: transaction.block_number + ) found_token_transfers = Etherscan.list_token_transfers(address1.hash, nil) @@ -884,7 +906,12 @@ defmodule Explorer.EtherscanTest do |> insert() |> with_block() - token_transfer = insert(:token_transfer, transaction: transaction) + token_transfer = + insert(:token_transfer, + transaction: transaction, + block: transaction.block, + block_number: transaction.block_number + ) insert(:block) @@ -903,7 +930,12 @@ defmodule Explorer.EtherscanTest do |> insert() |> with_block() - token_transfer = insert(:token_transfer, transaction: transaction) + token_transfer = + insert(:token_transfer, + transaction: transaction, + block: transaction.block, + block_number: transaction.block_number + ) {:ok, token} = Chain.token_from_address_hash(token_transfer.token_contract_address_hash) @@ -1019,11 +1051,29 @@ defmodule Explorer.EtherscanTest do |> insert() |> with_block(third_block) - second_block_token_transfers = insert_list(2, :token_transfer, from_address: address, transaction: transaction2) + second_block_token_transfers = + insert_list(2, :token_transfer, + from_address: address, + transaction: transaction2, + block: transaction2.block, + block_number: transaction2.block_number + ) - first_block_token_transfers = insert_list(2, :token_transfer, from_address: address, transaction: transaction3) + first_block_token_transfers = + insert_list(2, :token_transfer, + from_address: address, + transaction: transaction3, + block: transaction3.block, + block_number: transaction3.block_number + ) - third_block_token_transfers = insert_list(2, :token_transfer, from_address: address, transaction: transaction1) + third_block_token_transfers = + insert_list(2, :token_transfer, + from_address: address, + transaction: transaction1, + block: transaction1.block, + block_number: transaction1.block_number + ) options1 = %{page_number: 1, page_size: 2} @@ -1076,7 +1126,12 @@ defmodule Explorer.EtherscanTest do |> insert() |> with_block(block) - insert(:token_transfer, from_address: address, transaction: transaction) + insert(:token_transfer, + from_address: address, + transaction: transaction, + block: block, + block_number: block.number + ) end options = %{ @@ -1105,7 +1160,12 @@ defmodule Explorer.EtherscanTest do |> insert() |> with_block(block) - insert(:token_transfer, from_address: address, transaction: transaction) + insert(:token_transfer, + from_address: address, + transaction: transaction, + block: block, + block_number: block.number + ) end options = %{start_block: third_block.number} @@ -1131,7 +1191,12 @@ defmodule Explorer.EtherscanTest do |> insert() |> with_block(block) - insert(:token_transfer, from_address: address, transaction: transaction) + insert(:token_transfer, + from_address: address, + transaction: transaction, + block: block, + block_number: block.number + ) end options = %{end_block: second_block.number} @@ -1160,7 +1225,14 @@ defmodule Explorer.EtherscanTest do |> with_block() insert(:token_transfer, from_address: address, transaction: transaction) - insert(:token_transfer, from_address: address, token_contract_address: contract_address, transaction: transaction) + + insert(:token_transfer, + from_address: address, + token_contract_address: contract_address, + transaction: transaction, + block: transaction.block, + block_number: transaction.block_number + ) [found_token_transfer] = Etherscan.list_token_transfers(address.hash, contract_address.hash) From 5ad684dc5e63e788e22586dd01c67126a3d0e377 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 26 Nov 2019 17:32:53 +0300 Subject: [PATCH 13/57] fix joining --- apps/explorer/lib/explorer/etherscan.ex | 7 ++++--- apps/explorer/test/explorer/etherscan_test.exs | 7 ++++++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/apps/explorer/lib/explorer/etherscan.ex b/apps/explorer/lib/explorer/etherscan.ex index 90b3b96324..41d8df9cb3 100644 --- a/apps/explorer/lib/explorer/etherscan.ex +++ b/apps/explorer/lib/explorer/etherscan.ex @@ -8,7 +8,7 @@ defmodule Explorer.Etherscan do alias Explorer.Etherscan.Logs alias Explorer.{Chain, Repo} alias Explorer.Chain.Address.TokenBalance - alias Explorer.Chain.{Block, Hash, InternalTransaction, Transaction} + alias Explorer.Chain.{Block, Hash, InternalTransaction, TokenTransfer, Transaction} @default_options %{ order_by_direction: :desc, @@ -318,10 +318,11 @@ defmodule Explorer.Etherscan do query = from( t in Transaction, - inner_join: tt in assoc(t, :token_transfers), + inner_join: tt in TokenTransfer, + on: tt.transaction_hash == t.hash and tt.block_number == t.block_number and tt.block_hash == t.block_hash, inner_join: tkn in assoc(tt, :token), inner_join: b in assoc(t, :block), - where: tt.from_address_hash == ^address_hash and b.number == tt.block_number and b.hash == t.block_hash, + where: tt.from_address_hash == ^address_hash, or_where: tt.to_address_hash == ^address_hash, order_by: [{^options.order_by_direction, t.block_number}], limit: ^options.page_size, diff --git a/apps/explorer/test/explorer/etherscan_test.exs b/apps/explorer/test/explorer/etherscan_test.exs index 5ba4b8dc8e..7c1e07a438 100644 --- a/apps/explorer/test/explorer/etherscan_test.exs +++ b/apps/explorer/test/explorer/etherscan_test.exs @@ -846,7 +846,12 @@ defmodule Explorer.EtherscanTest do |> insert() |> with_block() - token_transfer = insert(:token_transfer, transaction: transaction) + token_transfer = + insert(:token_transfer, + transaction: transaction, + block: transaction.block, + block_number: transaction.block_number + ) [found_token_transfer] = Etherscan.list_token_transfers(token_transfer.to_address_hash, nil) From f488cfa98275fa415da0a78241e11044ace0a6ec Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Wed, 27 Nov 2019 10:51:22 +0300 Subject: [PATCH 14/57] remove unique index on transaction_hash, log_index for token transfers --- .../20191122062035_add_block_hash_to_token_transfers.exs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/explorer/priv/repo/migrations/20191122062035_add_block_hash_to_token_transfers.exs b/apps/explorer/priv/repo/migrations/20191122062035_add_block_hash_to_token_transfers.exs index c3ee02f605..795c8c28cb 100644 --- a/apps/explorer/priv/repo/migrations/20191122062035_add_block_hash_to_token_transfers.exs +++ b/apps/explorer/priv/repo/migrations/20191122062035_add_block_hash_to_token_transfers.exs @@ -29,5 +29,7 @@ defmodule Explorer.Repo.Migrations.AddBlockHashToTokenTransfers do DROP CONSTRAINT token_transfers_pkey, ADD PRIMARY KEY (transaction_hash, block_hash, log_index); """) + + drop(unique_index(:token_transfers, [:transaction_hash, :log_index])) end end From a6c9f086156b3c53dcf363b98f2dad2dc239b26f Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Wed, 27 Nov 2019 11:12:44 +0300 Subject: [PATCH 15/57] create regular index --- .../20191122062035_add_block_hash_to_token_transfers.exs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/explorer/priv/repo/migrations/20191122062035_add_block_hash_to_token_transfers.exs b/apps/explorer/priv/repo/migrations/20191122062035_add_block_hash_to_token_transfers.exs index 795c8c28cb..5635cd9bd9 100644 --- a/apps/explorer/priv/repo/migrations/20191122062035_add_block_hash_to_token_transfers.exs +++ b/apps/explorer/priv/repo/migrations/20191122062035_add_block_hash_to_token_transfers.exs @@ -31,5 +31,7 @@ defmodule Explorer.Repo.Migrations.AddBlockHashToTokenTransfers do """) drop(unique_index(:token_transfers, [:transaction_hash, :log_index])) + + create_if_not_exists(index(:token_transfers, [:transaction_hash, :log_index])) end end From 6a933d514f001a581f4959159afad56b152e0eef Mon Sep 17 00:00:00 2001 From: pasqu4le Date: Tue, 29 Oct 2019 18:12:14 +0100 Subject: [PATCH 16/57] internal_txs --- .../lib/block_scout_web/notifier.ex | 2 +- .../channels/address_channel_test.exs | 28 +- .../address_contract_controller_test.exs | 6 +- ...s_internal_transaction_controller_test.exs | 56 +++- .../address_read_contract_controller_test.exs | 6 +- .../address_transaction_controller_test.exs | 4 +- .../api/rpc/address_controller_test.exs | 35 ++- .../api/rpc/transaction_controller_test.exs | 10 +- ...n_internal_transaction_controller_test.exs | 28 +- .../features/viewing_addresses_test.exs | 28 +- .../features/viewing_app_test.exs | 11 +- .../features/viewing_blocks_test.exs | 7 +- .../features/viewing_transactions_test.exs | 10 +- .../schema/query/node_test.exs | 22 +- .../schema/query/transaction_test.exs | 63 +++- .../views/address_view_test.exs | 6 +- .../views/transaction_view_test.exs | 4 +- apps/explorer/lib/explorer/chain.ex | 140 ++------- apps/explorer/lib/explorer/chain/block.ex | 5 +- apps/explorer/lib/explorer/chain/import.ex | 3 +- .../explorer/chain/import/runner/blocks.ex | 61 ++-- .../import/runner/internal_transactions.ex | 269 ++++++++++++----- ...internal_transactions_indexed_at_blocks.ex | 82 ------ .../chain/import/runner/transactions.ex | 50 +--- .../chain/import/stage/block_following.ex | 2 - .../chain/import/stage/block_pending.ex | 26 ++ .../explorer/chain/internal_transaction.ex | 81 ++++- .../explorer/chain/pending_block_operation.ex | 45 ++- .../lib/explorer/chain/transaction.ex | 7 +- apps/explorer/lib/explorer/etherscan.ex | 2 + apps/explorer/lib/explorer/etherscan/logs.ex | 2 +- apps/explorer/lib/explorer/graphql.ex | 6 +- ...054_add_pending_internal_txs_operation.exs | 76 +++++ .../chain/cache/pending_transactions_test.exs | 4 +- .../test/explorer/chain/cache/uncles_test.exs | 1 - .../chain/import/runner/blocks_test.exs | 20 +- .../runner/internal_transactions_test.exs | 80 +++-- .../test/explorer/chain/import_test.exs | 101 ++----- .../chain/internal_transaction_test.exs | 4 +- apps/explorer/test/explorer/chain_test.exs | 105 ++++++- .../explorer/test/explorer/etherscan_test.exs | 30 ++ apps/explorer/test/explorer/graphql_test.exs | 77 ++++- apps/explorer/test/explorer/repo_test.exs | 8 +- apps/explorer/test/support/factory.ex | 22 +- .../lib/indexer/block/catchup/fetcher.ex | 12 +- apps/indexer/lib/indexer/block/fetcher.ex | 31 +- .../lib/indexer/block/realtime/fetcher.ex | 12 +- .../indexer/fetcher/internal_transaction.ex | 194 ++++-------- .../test/indexer/block/fetcher_test.exs | 9 +- .../fetcher/internal_transaction_test.exs | 278 ++++++++---------- 50 files changed, 1237 insertions(+), 934 deletions(-) delete mode 100644 apps/explorer/lib/explorer/chain/import/runner/internal_transactions_indexed_at_blocks.ex create mode 100644 apps/explorer/lib/explorer/chain/import/stage/block_pending.ex create mode 100644 apps/explorer/priv/repo/migrations/20191018140054_add_pending_internal_txs_operation.exs diff --git a/apps/block_scout_web/lib/block_scout_web/notifier.ex b/apps/block_scout_web/lib/block_scout_web/notifier.ex index 4bf9185d23..e0f21b528e 100644 --- a/apps/block_scout_web/lib/block_scout_web/notifier.ex +++ b/apps/block_scout_web/lib/block_scout_web/notifier.ex @@ -95,7 +95,7 @@ defmodule BlockScoutWeb.Notifier do def handle_event({:chain_event, :internal_transactions, :realtime, internal_transactions}) do internal_transactions |> Stream.map( - &(InternalTransaction + &(InternalTransaction.where_nonpending_block() |> Repo.get_by(transaction_hash: &1.transaction_hash, index: &1.index) |> Repo.preload([:from_address, :to_address, transaction: :block])) ) diff --git a/apps/block_scout_web/test/block_scout_web/channels/address_channel_test.exs b/apps/block_scout_web/test/block_scout_web/channels/address_channel_test.exs index 357a73eba8..6c830f2e10 100644 --- a/apps/block_scout_web/test/block_scout_web/channels/address_channel_test.exs +++ b/apps/block_scout_web/test/block_scout_web/channels/address_channel_test.exs @@ -134,7 +134,15 @@ defmodule BlockScoutWeb.AddressChannelTest do |> insert(from_address: address) |> with_block() - internal_transaction = insert(:internal_transaction, transaction: transaction, from_address: address, index: 0) + internal_transaction = + insert( + :internal_transaction, + transaction: transaction, + from_address: address, + index: 0, + block_hash: transaction.block_hash, + block_index: 0 + ) Notifier.handle_event({:chain_event, :internal_transactions, :realtime, [internal_transaction]}) @@ -158,7 +166,14 @@ defmodule BlockScoutWeb.AddressChannelTest do |> insert(to_address: address) |> with_block() - internal_transaction = insert(:internal_transaction, transaction: transaction, to_address: address, index: 0) + internal_transaction = + insert(:internal_transaction, + transaction: transaction, + to_address: address, + index: 0, + block_hash: transaction.block_hash, + block_index: 0 + ) Notifier.handle_event({:chain_event, :internal_transactions, :realtime, [internal_transaction]}) @@ -186,7 +201,14 @@ defmodule BlockScoutWeb.AddressChannelTest do |> with_block() internal_transaction = - insert(:internal_transaction, transaction: transaction, from_address: address, to_address: address, index: 0) + insert(:internal_transaction, + transaction: transaction, + from_address: address, + to_address: address, + index: 0, + block_hash: transaction.block_hash, + block_index: 0 + ) Notifier.handle_event({:chain_event, :internal_transactions, :realtime, [internal_transaction]}) diff --git a/apps/block_scout_web/test/block_scout_web/controllers/address_contract_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/address_contract_controller_test.exs index 1d5d00ab95..808b153e59 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/address_contract_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/address_contract_controller_test.exs @@ -35,13 +35,15 @@ defmodule BlockScoutWeb.AddressContractControllerTest do test "successfully renders the page when the address is a contract", %{conn: conn} do address = insert(:address, contract_code: Factory.data("contract_code"), smart_contract: nil) - transaction = insert(:transaction, from_address: address) + transaction = insert(:transaction, from_address: address) |> with_block() insert( :internal_transaction_create, index: 0, transaction: transaction, - created_contract_address: address + created_contract_address: address, + block_hash: transaction.block_hash, + block_index: 0 ) conn = get(conn, address_contract_path(BlockScoutWeb.Endpoint, :index, address)) diff --git a/apps/block_scout_web/test/block_scout_web/controllers/address_internal_transaction_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/address_internal_transaction_controller_test.exs index 90b0c6500f..274f4c6ba7 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/address_internal_transaction_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/address_internal_transaction_controller_test.exs @@ -44,7 +44,9 @@ defmodule BlockScoutWeb.AddressInternalTransactionControllerTest do from_address: address, index: 1, block_number: transaction.block_number, - transaction_index: transaction.index + transaction_index: transaction.index, + block_hash: transaction.block_hash, + block_index: 1 ) to_internal_transaction = @@ -53,7 +55,9 @@ defmodule BlockScoutWeb.AddressInternalTransactionControllerTest do to_address: address, index: 2, block_number: transaction.block_number, - transaction_index: transaction.index + transaction_index: transaction.index, + block_hash: transaction.block_hash, + block_index: 2 ) path = address_internal_transaction_path(conn, :index, address, %{"type" => "JSON"}) @@ -83,7 +87,9 @@ defmodule BlockScoutWeb.AddressInternalTransactionControllerTest do from_address: address, index: 1, block_number: transaction.block_number, - transaction_index: transaction.index + transaction_index: transaction.index, + block_hash: transaction.block_hash, + block_index: 1 ) to_internal_transaction = @@ -92,7 +98,9 @@ defmodule BlockScoutWeb.AddressInternalTransactionControllerTest do to_address: address, index: 2, block_number: transaction.block_number, - transaction_index: transaction.index + transaction_index: transaction.index, + block_hash: transaction.block_hash, + block_index: 2 ) path = address_internal_transaction_path(conn, :index, address, %{"filter" => "from", "type" => "JSON"}) @@ -125,7 +133,9 @@ defmodule BlockScoutWeb.AddressInternalTransactionControllerTest do from_address: address, index: 1, block_number: transaction.block_number, - transaction_index: transaction.index + transaction_index: transaction.index, + block_hash: transaction.block_hash, + block_index: 1 ) to_internal_transaction = @@ -134,7 +144,9 @@ defmodule BlockScoutWeb.AddressInternalTransactionControllerTest do to_address: address, index: 2, block_number: transaction.block_number, - transaction_index: transaction.index + transaction_index: transaction.index, + block_hash: transaction.block_hash, + block_index: 2 ) path = address_internal_transaction_path(conn, :index, address, %{"filter" => "to", "type" => "JSON"}) @@ -167,7 +179,9 @@ defmodule BlockScoutWeb.AddressInternalTransactionControllerTest do from_address: address, index: 1, block_number: transaction.block_number, - transaction_index: transaction.index + transaction_index: transaction.index, + block_hash: transaction.block_hash, + block_index: 1 ) to_internal_transaction = @@ -177,7 +191,9 @@ defmodule BlockScoutWeb.AddressInternalTransactionControllerTest do created_contract_address: address, index: 2, block_number: transaction.block_number, - transaction_index: transaction.index + transaction_index: transaction.index, + block_hash: transaction.block_hash, + block_index: 2 ) path = address_internal_transaction_path(conn, :index, address, %{"filter" => "to", "type" => "JSON"}) @@ -226,7 +242,9 @@ defmodule BlockScoutWeb.AddressInternalTransactionControllerTest do from_address: address, index: index, block_number: transaction_1.block_number, - transaction_index: transaction_1.index + transaction_index: transaction_1.index, + block_hash: a_block.hash, + block_index: index ) end) @@ -239,7 +257,9 @@ defmodule BlockScoutWeb.AddressInternalTransactionControllerTest do from_address: address, index: index, block_number: transaction_2.block_number, - transaction_index: transaction_2.index + transaction_index: transaction_2.index, + block_hash: a_block.hash, + block_index: 20 + index ) end) @@ -252,7 +272,9 @@ defmodule BlockScoutWeb.AddressInternalTransactionControllerTest do from_address: address, index: index, block_number: transaction_3.block_number, - transaction_index: transaction_3.index + transaction_index: transaction_3.index, + block_hash: b_block.hash, + block_index: index ) end) @@ -265,7 +287,9 @@ defmodule BlockScoutWeb.AddressInternalTransactionControllerTest do from_address: address, index: 11, block_number: transaction_3.block_number, - transaction_index: transaction_3.index + transaction_index: transaction_3.index, + block_hash: b_block.hash, + block_index: 11 ) conn = @@ -304,7 +328,9 @@ defmodule BlockScoutWeb.AddressInternalTransactionControllerTest do from_address: address, index: index, block_number: transaction.block_number, - transaction_index: transaction.index + transaction_index: transaction.index, + block_hash: transaction.block_hash, + block_index: index ) end) @@ -335,7 +361,9 @@ defmodule BlockScoutWeb.AddressInternalTransactionControllerTest do :internal_transaction, transaction: transaction, from_address: address, - index: index + index: index, + block_hash: transaction.block_hash, + block_index: index ) end) diff --git a/apps/block_scout_web/test/block_scout_web/controllers/address_read_contract_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/address_read_contract_controller_test.exs index 0e66980768..ac3e39822b 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/address_read_contract_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/address_read_contract_controller_test.exs @@ -21,13 +21,15 @@ defmodule BlockScoutWeb.AddressReadContractControllerTest do test "successfully renders the page when the address is a contract", %{conn: conn} do contract_address = insert(:contract_address) - transaction = insert(:transaction, from_address: contract_address) + transaction = insert(:transaction, from_address: contract_address) |> with_block() insert( :internal_transaction_create, index: 0, transaction: transaction, - created_contract_address: contract_address + created_contract_address: contract_address, + block_hash: transaction.block_hash, + block_index: 0 ) insert(:smart_contract, address_hash: contract_address.hash) diff --git a/apps/block_scout_web/test/block_scout_web/controllers/address_transaction_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/address_transaction_controller_test.exs index ccffa5672d..224e5d1db4 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/address_transaction_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/address_transaction_controller_test.exs @@ -126,7 +126,9 @@ defmodule BlockScoutWeb.AddressTransactionControllerTest do index: 0, created_contract_address: address, to_address: nil, - transaction: transaction + transaction: transaction, + block_hash: block.hash, + block_index: 0 ) conn = get(conn, address_transaction_path(conn, :index, address), %{"type" => "JSON"}) diff --git a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/address_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/address_controller_test.exs index 2215057a9a..4c2539268f 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/address_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/address_controller_test.exs @@ -1453,7 +1453,13 @@ defmodule BlockScoutWeb.API.RPC.AddressControllerTest do internal_transaction = :internal_transaction_create - |> insert(transaction: transaction, index: 0, from_address: address) + |> insert( + transaction: transaction, + index: 0, + from_address: address, + block_hash: transaction.block_hash, + block_index: 0 + ) |> with_contract_creation(contract_address) params = %{ @@ -1502,7 +1508,9 @@ defmodule BlockScoutWeb.API.RPC.AddressControllerTest do transaction: transaction, index: 0, type: :reward, - error: "some error" + error: "some error", + block_hash: transaction.block_hash, + block_index: 0 ] insert(:internal_transaction_create, internal_transaction_details) @@ -1532,7 +1540,12 @@ defmodule BlockScoutWeb.API.RPC.AddressControllerTest do |> with_block() for index <- 0..2 do - insert(:internal_transaction_create, transaction: transaction, index: index) + insert(:internal_transaction_create, + transaction: transaction, + index: index, + block_hash: transaction.block_hash, + block_index: index + ) end params = %{ @@ -1606,7 +1619,13 @@ defmodule BlockScoutWeb.API.RPC.AddressControllerTest do internal_transaction = :internal_transaction_create - |> insert(transaction: transaction, index: 0, from_address: address) + |> insert( + transaction: transaction, + index: 0, + from_address: address, + block_hash: transaction.block_hash, + block_index: 0 + ) |> with_contract_creation(contract_address) params = %{ @@ -1658,7 +1677,9 @@ defmodule BlockScoutWeb.API.RPC.AddressControllerTest do transaction: transaction, index: 0, type: :reward, - error: "some error" + error: "some error", + block_hash: transaction.block_hash, + block_index: 0 ] insert(:internal_transaction_create, internal_transaction_details) @@ -1693,7 +1714,9 @@ defmodule BlockScoutWeb.API.RPC.AddressControllerTest do internal_transaction_details = %{ from_address: address, transaction: transaction, - index: index + index: index, + block_hash: transaction.block_hash, + block_index: index } insert(:internal_transaction_create, internal_transaction_details) diff --git a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/transaction_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/transaction_controller_test.exs index 0ab9743bf7..d83e530c28 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/transaction_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/transaction_controller_test.exs @@ -243,8 +243,7 @@ defmodule BlockScoutWeb.API.RPC.TransactionControllerTest do transaction_details = [ status: :error, - error: error, - internal_transactions_indexed_at: DateTime.utc_now() + error: error ] transaction = @@ -256,7 +255,9 @@ defmodule BlockScoutWeb.API.RPC.TransactionControllerTest do transaction: transaction, index: 0, type: :reward, - error: error + error: error, + block_hash: transaction.block_hash, + block_index: 0 ] insert(:internal_transaction, internal_transaction_details) @@ -285,8 +286,7 @@ defmodule BlockScoutWeb.API.RPC.TransactionControllerTest do test "with a txhash with failed status but awaiting internal transactions", %{conn: conn} do transaction_details = [ status: :error, - error: nil, - internal_transactions_indexed_at: nil + error: nil ] transaction = diff --git a/apps/block_scout_web/test/block_scout_web/controllers/transaction_internal_transaction_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/transaction_internal_transaction_controller_test.exs index 773b51a36c..aff5ca32d5 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/transaction_internal_transaction_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/transaction_internal_transaction_controller_test.exs @@ -44,14 +44,18 @@ defmodule BlockScoutWeb.TransactionInternalTransactionControllerTest do transaction: transaction, index: 0, block_number: transaction.block_number, - transaction_index: transaction.index + transaction_index: transaction.index, + block_hash: transaction.block_hash, + block_index: 0 ) insert(:internal_transaction, transaction: transaction, index: 1, transaction_index: transaction.index, - block_number: transaction.block_number + block_number: transaction.block_number, + block_hash: transaction.block_hash, + block_index: 1 ) path = transaction_internal_transaction_path(BlockScoutWeb.Endpoint, :index, transaction.hash) @@ -90,7 +94,9 @@ defmodule BlockScoutWeb.TransactionInternalTransactionControllerTest do transaction: transaction, index: 0, block_number: transaction.block_number, - transaction_index: transaction.index + transaction_index: transaction.index, + block_hash: transaction.block_hash, + block_index: 0 ) |> with_contract_creation(contract_address) @@ -118,7 +124,9 @@ defmodule BlockScoutWeb.TransactionInternalTransactionControllerTest do transaction: transaction, index: 0, block_number: transaction.block_number, - transaction_index: transaction.index + transaction_index: transaction.index, + block_hash: transaction.block_hash, + block_index: 0 ) second_page_indexes = @@ -128,7 +136,9 @@ defmodule BlockScoutWeb.TransactionInternalTransactionControllerTest do transaction: transaction, index: index, block_number: transaction.block_number, - transaction_index: transaction.index + transaction_index: transaction.index, + block_hash: transaction.block_hash, + block_index: index ) end) |> Enum.map(& &1.index) @@ -161,7 +171,9 @@ defmodule BlockScoutWeb.TransactionInternalTransactionControllerTest do transaction: transaction, index: index, block_number: transaction.block_number, - transaction_index: transaction.index + transaction_index: transaction.index, + block_hash: transaction.block_hash, + block_index: index ) end) @@ -190,7 +202,9 @@ defmodule BlockScoutWeb.TransactionInternalTransactionControllerTest do transaction: transaction, index: index, block_number: transaction.block_number, - transaction_index: transaction.index + transaction_index: transaction.index, + block_hash: transaction.block_hash, + block_index: index ) end) diff --git a/apps/block_scout_web/test/block_scout_web/features/viewing_addresses_test.exs b/apps/block_scout_web/test/block_scout_web/features/viewing_addresses_test.exs index 82b5498a69..0ad8671301 100644 --- a/apps/block_scout_web/test/block_scout_web/features/viewing_addresses_test.exs +++ b/apps/block_scout_web/test/block_scout_web/features/viewing_addresses_test.exs @@ -80,7 +80,7 @@ defmodule BlockScoutWeb.ViewingAddressesTest do test "see the contract creator and transaction links", %{session: session} do address = insert(:address) contract = insert(:contract_address) - transaction = insert(:transaction, from_address: address, created_contract_address: contract) + transaction = insert(:transaction, from_address: address, created_contract_address: contract) |> with_block() internal_transaction = insert( @@ -88,7 +88,9 @@ defmodule BlockScoutWeb.ViewingAddressesTest do index: 1, transaction: transaction, from_address: address, - created_contract_address: contract + created_contract_address: contract, + block_hash: transaction.block_hash, + block_index: 1 ) address_hash = AddressView.trimmed_hash(address.hash) @@ -102,7 +104,7 @@ defmodule BlockScoutWeb.ViewingAddressesTest do test "see the contract creator and transaction links even when the creator is another contract", %{session: session} do lincoln = insert(:address) contract = insert(:contract_address) - transaction = insert(:transaction) + transaction = insert(:transaction) |> with_block() another_contract = insert(:contract_address) insert( @@ -112,7 +114,9 @@ defmodule BlockScoutWeb.ViewingAddressesTest do from_address: lincoln, to_address: contract, created_contract_address: contract, - type: :call + type: :call, + block_hash: transaction.block_hash, + block_index: 1 ) internal_transaction = @@ -121,7 +125,9 @@ defmodule BlockScoutWeb.ViewingAddressesTest do index: 2, transaction: transaction, from_address: contract, - created_contract_address: another_contract + created_contract_address: another_contract, + block_hash: transaction.block_hash, + block_index: 2 ) contract_hash = AddressView.trimmed_hash(contract.hash) @@ -208,7 +214,9 @@ defmodule BlockScoutWeb.ViewingAddressesTest do to_address: address, index: 1, block_number: 7000, - transaction_index: 1 + transaction_index: 1, + block_hash: transaction.block_hash, + block_index: 1 ) insert(:internal_transaction, @@ -216,7 +224,9 @@ defmodule BlockScoutWeb.ViewingAddressesTest do from_address: address, index: 2, block_number: 8000, - transaction_index: 2 + transaction_index: 2, + block_hash: transaction.block_hash, + block_index: 2 ) {:ok, %{internal_transaction_lincoln_to_address: internal_transaction_lincoln_to_address}} @@ -251,7 +261,9 @@ defmodule BlockScoutWeb.ViewingAddressesTest do index: 2, from_address: addresses.lincoln, block_number: transaction.block_number, - transaction_index: transaction.index + transaction_index: transaction.index, + block_hash: transaction.block_hash, + block_index: 2 ) Notifier.handle_event({:chain_event, :internal_transactions, :realtime, [internal_transaction]}) diff --git a/apps/block_scout_web/test/block_scout_web/features/viewing_app_test.exs b/apps/block_scout_web/test/block_scout_web/features/viewing_app_test.exs index ff37eed468..d7d7777d16 100644 --- a/apps/block_scout_web/test/block_scout_web/features/viewing_app_test.exs +++ b/apps/block_scout_web/test/block_scout_web/features/viewing_app_test.exs @@ -7,7 +7,7 @@ defmodule BlockScoutWeb.ViewingAppTest do alias BlockScoutWeb.Counters.BlocksIndexedCounter alias Explorer.Counters.AddressesCounter alias Explorer.{Repo} - alias Explorer.Chain.{Transaction} + alias Explorer.Chain.{PendingBlockOperation, Transaction} setup do start_supervised!(AddressesCounter) @@ -111,6 +111,10 @@ defmodule BlockScoutWeb.ViewingAppTest do |> insert() |> with_block(block) + block_hash = block.hash + + insert(:pending_block_operation, block_hash: block_hash, fetch_internal_transactions: true) + BlocksIndexedCounter.calculate_blocks_indexed() assert Decimal.cmp(Explorer.Chain.indexed_ratio(), 1) == :eq @@ -119,7 +123,10 @@ defmodule BlockScoutWeb.ViewingAppTest do |> AppPage.visit_page() |> assert_has(AppPage.indexed_status("Indexing Tokens")) - Repo.update_all(Transaction, set: [internal_transactions_indexed_at: DateTime.utc_now()]) + Repo.update_all( + from(p in PendingBlockOperation, where: p.block_hash == ^block_hash), + set: [fetch_internal_transactions: false] + ) BlocksIndexedCounter.calculate_blocks_indexed() diff --git a/apps/block_scout_web/test/block_scout_web/features/viewing_blocks_test.exs b/apps/block_scout_web/test/block_scout_web/features/viewing_blocks_test.exs index 208fe1bd65..8812b65d8a 100644 --- a/apps/block_scout_web/test/block_scout_web/features/viewing_blocks_test.exs +++ b/apps/block_scout_web/test/block_scout_web/features/viewing_blocks_test.exs @@ -59,7 +59,12 @@ defmodule BlockScoutWeb.ViewingBlocksTest do internal_transaction = :internal_transaction_create - |> insert(transaction: transaction, index: 0) + |> insert( + transaction: transaction, + index: 0, + block_hash: transaction.block_hash, + block_index: 1 + ) |> with_contract_creation(contract_address) session diff --git a/apps/block_scout_web/test/block_scout_web/features/viewing_transactions_test.exs b/apps/block_scout_web/test/block_scout_web/features/viewing_transactions_test.exs index b72431fb41..26b7e07a73 100644 --- a/apps/block_scout_web/test/block_scout_web/features/viewing_transactions_test.exs +++ b/apps/block_scout_web/test/block_scout_web/features/viewing_transactions_test.exs @@ -46,7 +46,13 @@ defmodule BlockScoutWeb.ViewingTransactionsTest do insert(:log, address: lincoln, index: 0, transaction: transaction) - internal = insert(:internal_transaction, index: 0, transaction: transaction) + internal = + insert(:internal_transaction, + index: 0, + transaction: transaction, + block_hash: transaction.block_hash, + block_index: 0 + ) {:ok, %{ @@ -95,7 +101,7 @@ defmodule BlockScoutWeb.ViewingTransactionsTest do |> with_contract_creation(contract_address) :internal_transaction_create - |> insert(transaction: transaction, index: 0) + |> insert(transaction: transaction, index: 0, block_hash: transaction.block_hash, block_index: 0) |> with_contract_creation(contract_address) session diff --git a/apps/block_scout_web/test/block_scout_web/schema/query/node_test.exs b/apps/block_scout_web/test/block_scout_web/schema/query/node_test.exs index a8330f085a..95acf02a61 100644 --- a/apps/block_scout_web/test/block_scout_web/schema/query/node_test.exs +++ b/apps/block_scout_web/test/block_scout_web/schema/query/node_test.exs @@ -58,9 +58,15 @@ defmodule BlockScoutWeb.Schema.Query.NodeTest do end test "with valid argument 'id' for an internal transaction", %{conn: conn} do - transaction = insert(:transaction) + transaction = insert(:transaction) |> with_block() - internal_transaction = insert(:internal_transaction, transaction: transaction, index: 0) + internal_transaction = + insert(:internal_transaction, + transaction: transaction, + index: 0, + block_hash: transaction.block_hash, + block_index: 0 + ) query = """ query($id: ID!) { @@ -96,9 +102,15 @@ defmodule BlockScoutWeb.Schema.Query.NodeTest do end test "with 'id' for non-existent internal transaction", %{conn: conn} do - transaction = build(:transaction) - - internal_transaction = build(:internal_transaction, transaction: transaction, index: 0) + transaction = build(:transaction) |> with_block() + + internal_transaction = + build(:internal_transaction, + transaction: transaction, + index: 0, + block_hash: transaction.block_hash, + block_index: 0 + ) query = """ query($id: ID!) { diff --git a/apps/block_scout_web/test/block_scout_web/schema/query/transaction_test.exs b/apps/block_scout_web/test/block_scout_web/schema/query/transaction_test.exs index b647e69de6..60f6b423b4 100644 --- a/apps/block_scout_web/test/block_scout_web/schema/query/transaction_test.exs +++ b/apps/block_scout_web/test/block_scout_web/schema/query/transaction_test.exs @@ -134,7 +134,9 @@ defmodule BlockScoutWeb.Schema.Query.TransactionTest do transaction: transaction, index: 0, from_address: address, - call_type: :call + call_type: :call, + block_hash: transaction.block_hash, + block_index: 0 } internal_transaction = @@ -259,11 +261,28 @@ defmodule BlockScoutWeb.Schema.Query.TransactionTest do end test "internal transactions are ordered by ascending index", %{conn: conn} do - transaction = insert(:transaction) + transaction = insert(:transaction) |> with_block() + + insert(:internal_transaction, + transaction: transaction, + index: 2, + block_hash: transaction.block_hash, + block_index: 2 + ) + + insert(:internal_transaction, + transaction: transaction, + index: 0, + block_hash: transaction.block_hash, + block_index: 0 + ) - insert(:internal_transaction, transaction: transaction, index: 2) - insert(:internal_transaction, transaction: transaction, index: 0) - insert(:internal_transaction, transaction: transaction, index: 1) + insert(:internal_transaction, + transaction: transaction, + index: 1, + block_hash: transaction.block_hash, + block_index: 1 + ) query = """ query ($hash: FullHash!, $first: Int!) { @@ -370,11 +389,28 @@ defmodule BlockScoutWeb.Schema.Query.TransactionTest do # # This test ensures support for a 'count' argument. - transaction = insert(:transaction) + transaction = insert(:transaction) |> with_block() - insert(:internal_transaction, transaction: transaction, index: 2) - insert(:internal_transaction, transaction: transaction, index: 0) - insert(:internal_transaction, transaction: transaction, index: 1) + insert(:internal_transaction, + transaction: transaction, + index: 2, + block_hash: transaction.block_hash, + block_index: 2 + ) + + insert(:internal_transaction, + transaction: transaction, + index: 0, + block_hash: transaction.block_hash, + block_index: 0 + ) + + insert(:internal_transaction, + transaction: transaction, + index: 1, + block_hash: transaction.block_hash, + block_index: 1 + ) query = """ query ($hash: FullHash!, $last: Int!, $count: Int!) { @@ -407,10 +443,15 @@ defmodule BlockScoutWeb.Schema.Query.TransactionTest do end test "pagination support with 'first' and 'after' arguments", %{conn: conn} do - transaction = insert(:transaction) + transaction = insert(:transaction) |> with_block() for index <- 0..5 do - insert(:internal_transaction_create, transaction: transaction, index: index) + insert(:internal_transaction_create, + transaction: transaction, + index: index, + block_hash: transaction.block_hash, + block_index: index + ) end query1 = """ diff --git a/apps/block_scout_web/test/block_scout_web/views/address_view_test.exs b/apps/block_scout_web/test/block_scout_web/views/address_view_test.exs index 97544754ec..417e51a6f5 100644 --- a/apps/block_scout_web/test/block_scout_web/views/address_view_test.exs +++ b/apps/block_scout_web/test/block_scout_web/views/address_view_test.exs @@ -11,14 +11,16 @@ defmodule BlockScoutWeb.AddressViewTest do end test "for a pending internal transaction contract creation to address" do - transaction = insert(:transaction, to_address: nil) + transaction = insert(:transaction, to_address: nil) |> with_block() internal_transaction = insert(:internal_transaction, index: 1, transaction: transaction, to_address: nil, - created_contract_address_hash: nil + created_contract_address_hash: nil, + block_hash: transaction.block_hash, + block_index: 1 ) assert "Contract Address Pending" == AddressView.address_partial_selector(internal_transaction, :to, nil) diff --git a/apps/block_scout_web/test/block_scout_web/views/transaction_view_test.exs b/apps/block_scout_web/test/block_scout_web/views/transaction_view_test.exs index e8505b144e..e8acf34c88 100644 --- a/apps/block_scout_web/test/block_scout_web/views/transaction_view_test.exs +++ b/apps/block_scout_web/test/block_scout_web/views/transaction_view_test.exs @@ -192,6 +192,8 @@ defmodule BlockScoutWeb.TransactionViewTest do |> insert() |> with_block(block, status: :error) + insert(:pending_block_operation, block_hash: block.hash, fetch_internal_transactions: true) + status = TransactionView.transaction_status(transaction) assert TransactionView.formatted_status(status) == "Error: (Awaiting internal transactions for reason)" end @@ -200,7 +202,7 @@ defmodule BlockScoutWeb.TransactionViewTest do transaction = :transaction |> insert() - |> with_block(status: :error, internal_transactions_indexed_at: DateTime.utc_now(), error: "Out of Gas") + |> with_block(status: :error, error: "Out of Gas") status = TransactionView.transaction_status(transaction) assert TransactionView.formatted_status(status) == "Error: Out of Gas" diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index 1475c97c99..74b910a711 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -194,6 +194,7 @@ defmodule Explorer.Chain do paging_options = Keyword.get(options, :paging_options, @default_paging_options) InternalTransaction + |> InternalTransaction.where_nonpending_block() |> InternalTransaction.where_address_fields_match(hash, direction) |> InternalTransaction.where_is_different_from_parent_transaction() |> InternalTransaction.where_block_number_is_not_null() @@ -718,8 +719,8 @@ defmodule Explorer.Chain do end @doc """ - Checks to see if the chain is down indexing based on the transaction from the oldest block having - an `internal_transactions_indexed_at` date. + Checks to see if the chain is down indexing based on the transaction from the + oldest block and the `fetch_internal_transactions` pending operation """ @spec finished_indexing?() :: boolean() def finished_indexing? do @@ -732,14 +733,15 @@ defmodule Explorer.Chain do if transaction_exists do if min_block_number_transaction do - Transaction - |> where([t], t.block_number == ^min_block_number_transaction and is_nil(t.internal_transactions_indexed_at)) - |> limit(1) - |> Repo.one() - |> case do - nil -> true - _ -> false - end + query = + from( + b in Block, + join: pending_ops in assoc(b, :pending_operations), + where: pending_ops.fetch_internal_transactions, + where: b.consensus and b.number == ^min_block_number_transaction + ) + + !Repo.exists?(query) else false end @@ -1340,7 +1342,7 @@ defmodule Explorer.Chain do ...> :transaction |> ...> insert() |> ...> with_block() - iex> insert(:internal_transaction, index: 0, transaction: transaction) + iex> insert(:internal_transaction, index: 0, transaction: transaction, block_hash: transaction.block_hash, block_index: 0) iex> Explorer.Chain.internal_transaction_count() 1 @@ -1351,7 +1353,8 @@ defmodule Explorer.Chain do """ def internal_transaction_count do - Repo.one!(from(it in "internal_transactions", select: fragment("COUNT(*)"))) + InternalTransaction.where_nonpending_block() + |> Repo.aggregate(:count, :transaction_hash) end @doc """ @@ -1632,13 +1635,14 @@ defmodule Explorer.Chain do Only blocks with consensus are returned. iex> non_consensus = insert(:block, consensus: false) + iex> insert(:pending_block_operation, block: non_consensus, fetch_internal_transactions: true) iex> unfetched = insert(:block) - iex> fetched = insert(:block, internal_transactions_indexed_at: DateTime.utc_now()) - iex> to_be_refetched = insert(:block, refetch_needed: true) + iex> insert(:pending_block_operation, block: unfetched, fetch_internal_transactions: true) + iex> fetched = insert(:block) + iex> insert(:pending_block_operation, block: fetched, fetch_internal_transactions: false) iex> {:ok, number_set} = Explorer.Chain.stream_blocks_with_unfetched_internal_transactions( - ...> [:number], ...> MapSet.new(), - ...> fn %Explorer.Chain.Block{number: number}, acc -> + ...> fn number, acc -> ...> MapSet.put(acc, number) ...> end ...> ) @@ -1648,103 +1652,21 @@ defmodule Explorer.Chain do true iex> fetched.hash in number_set false - iex> to_be_refetched.number in number_set - false """ @spec stream_blocks_with_unfetched_internal_transactions( - fields :: [ - :consensus - | :difficulty - | :gas_limit - | :gas_used - | :hash - | :miner - | :miner_hash - | :nonce - | :number - | :parent_hash - | :size - | :timestamp - | :total_difficulty - | :transactions - | :internal_transactions_indexed_at - ], initial :: accumulator, reducer :: (entry :: term(), accumulator -> accumulator) ) :: {:ok, accumulator} when accumulator: term() - def stream_blocks_with_unfetched_internal_transactions(fields, initial, reducer) when is_function(reducer, 2) do + def stream_blocks_with_unfetched_internal_transactions(initial, reducer) when is_function(reducer, 2) do query = from( b in Block, + join: pending_ops in assoc(b, :pending_operations), + where: pending_ops.fetch_internal_transactions, where: b.consensus, - where: is_nil(b.internal_transactions_indexed_at), - where: not b.refetch_needed, - select: ^fields - ) - - Repo.stream_reduce(query, initial, reducer) - end - - @doc """ - Returns a stream of all collated transactions with unfetched internal transactions. - - Only transactions that have been collated into a block are returned; pending transactions not in a block are filtered - out. - - iex> pending = insert(:transaction) - iex> unfetched_collated = - ...> :transaction |> - ...> insert() |> - ...> with_block() - iex> fetched_collated = - ...> :transaction |> - ...> insert() |> - ...> with_block(internal_transactions_indexed_at: DateTime.utc_now()) - iex> {:ok, hash_set} = Explorer.Chain.stream_transactions_with_unfetched_internal_transactions( - ...> [:hash], - ...> MapSet.new(), - ...> fn %Explorer.Chain.Transaction{hash: hash}, acc -> - ...> MapSet.put(acc, hash) - ...> end - ...> ) - iex> pending.hash in hash_set - false - iex> unfetched_collated.hash in hash_set - true - iex> fetched_collated.hash in hash_set - false - - """ - @spec stream_transactions_with_unfetched_internal_transactions( - fields :: [ - :block_hash - | :internal_transactions_indexed_at - | :from_address_hash - | :gas - | :gas_price - | :hash - | :index - | :input - | :nonce - | :r - | :s - | :to_address_hash - | :v - | :value - ], - initial :: accumulator, - reducer :: (entry :: term(), accumulator -> accumulator) - ) :: {:ok, accumulator} - when accumulator: term() - def stream_transactions_with_unfetched_internal_transactions(fields, initial, reducer) when is_function(reducer, 2) do - query = - from( - t in Transaction, - # exclude pending transactions and replaced transactions - where: not is_nil(t.block_hash) and is_nil(t.internal_transactions_indexed_at), - select: ^fields + select: b.number ) Repo.stream_reduce(query, initial, reducer) @@ -1753,7 +1675,6 @@ defmodule Explorer.Chain do @spec stream_transactions_with_unfetched_created_contract_codes( fields :: [ :block_hash - | :internal_transactions_indexed_at | :created_contract_code_indexed_at | :from_address_hash | :gas @@ -1788,7 +1709,6 @@ defmodule Explorer.Chain do @spec stream_mined_transactions( fields :: [ :block_hash - | :internal_transactions_indexed_at | :created_contract_code_indexed_at | :from_address_hash | :gas @@ -1820,7 +1740,6 @@ defmodule Explorer.Chain do @spec stream_pending_transactions( fields :: [ :block_hash - | :internal_transactions_indexed_at | :created_contract_code_indexed_at | :from_address_hash | :gas @@ -2202,8 +2121,8 @@ defmodule Explorer.Chain do ## Options * `:necessity_by_association` - use to load `t:association/0` as `:required` or `:optional`. If an association is - `:required`, and the `t:Explorer.Chain.InternalTransaction.t/0` has no associated record for that association, - then the `t:Explorer.Chain.InternalTransaction.t/0` will not be included in the list. + `:required`, and the `t:Explorer.Chain.Transaction.t/0` has no associated record for that association, + then the `t:Explorer.Chain.Transaction.t/0` will not be included in the list. * `:paging_options` - a `t:Explorer.PagingOptions.t/0` used to specify the `:page_size` and `:key` (a tuple of the lowest/oldest `{block_number, index}`) and. Results will be the transactions older than the `block_number` and `index` that are passed. @@ -2257,8 +2176,8 @@ defmodule Explorer.Chain do ## Options * `:necessity_by_association` - use to load `t:association/0` as `:required` or `:optional`. If an association is - `:required`, and the `t:Explorer.Chain.InternalTransaction.t/0` has no associated record for that association, - then the `t:Explorer.Chain.InternalTransaction.t/0` will not be included in the list. + `:required`, and the `t:Explorer.Chain.Transaction.t/0` has no associated record for that association, + then the `t:Explorer.Chain.Transaction.t/0` will not be included in the list. * `:paging_options` - a `t:Explorer.PagingOptions.t/0` used to specify the `:page_size` (defaults to `#{@default_paging_options.page_size}`) and `:key` (a tuple of the lowest/oldest `{inserted_at, hash}`) and. Results will be the transactions older than the `inserted_at` and `hash` that are passed. @@ -2438,6 +2357,7 @@ defmodule Explorer.Chain do |> for_parent_transaction(hash) |> join_associations(necessity_by_association) |> where_transaction_has_multiple_internal_transactions() + |> InternalTransaction.where_nonpending_block() |> page_internal_transaction(paging_options) |> limit(^paging_options.page_size) |> order_by([internal_transaction], asc: internal_transaction.index) @@ -2534,7 +2454,7 @@ defmodule Explorer.Chain do def transaction_to_status(%Transaction{status: nil}), do: :awaiting_internal_transactions def transaction_to_status(%Transaction{status: :ok}), do: :success - def transaction_to_status(%Transaction{status: :error, internal_transactions_indexed_at: nil, error: nil}), + def transaction_to_status(%Transaction{status: :error, error: nil}), do: {:error, :awaiting_internal_transactions} def transaction_to_status(%Transaction{status: :error, error: error}) when is_binary(error), do: {:error, error} diff --git a/apps/explorer/lib/explorer/chain/block.ex b/apps/explorer/lib/explorer/chain/block.ex index 0035eb2688..fa81b31993 100644 --- a/apps/explorer/lib/explorer/chain/block.ex +++ b/apps/explorer/lib/explorer/chain/block.ex @@ -10,7 +10,7 @@ defmodule Explorer.Chain.Block do alias Explorer.Chain.{Address, Gas, Hash, PendingBlockOperation, Transaction} alias Explorer.Chain.Block.{Reward, SecondDegreeRelation} - @optional_attrs ~w(internal_transactions_indexed_at size refetch_needed total_difficulty difficulty)a + @optional_attrs ~w(size refetch_needed total_difficulty difficulty)a @required_attrs ~w(consensus gas_limit gas_used hash miner_hash nonce number parent_hash timestamp)a @@ -46,7 +46,6 @@ defmodule Explorer.Chain.Block do * `timestamp` - When the block was collated * `total_difficulty` - the total `difficulty` of the chain until this block. * `transactions` - the `t:Explorer.Chain.Transaction.t/0` in this block. - * `internal_transactions_indexed_at` - when `internal_transactions` were fetched by `Indexer`. """ @type t :: %__MODULE__{ consensus: boolean(), @@ -63,7 +62,6 @@ defmodule Explorer.Chain.Block do timestamp: DateTime.t(), total_difficulty: difficulty(), transactions: %Ecto.Association.NotLoaded{} | [Transaction.t()], - internal_transactions_indexed_at: DateTime.t(), refetch_needed: boolean() } @@ -78,7 +76,6 @@ defmodule Explorer.Chain.Block do field(:size, :integer) field(:timestamp, :utc_datetime_usec) field(:total_difficulty, :decimal) - field(:internal_transactions_indexed_at, :utc_datetime_usec) field(:refetch_needed, :boolean) timestamps() diff --git a/apps/explorer/lib/explorer/chain/import.ex b/apps/explorer/lib/explorer/chain/import.ex index b682e1adf1..2aef04ae84 100644 --- a/apps/explorer/lib/explorer/chain/import.ex +++ b/apps/explorer/lib/explorer/chain/import.ex @@ -12,7 +12,8 @@ defmodule Explorer.Chain.Import do Import.Stage.Addresses, Import.Stage.AddressReferencing, Import.Stage.BlockReferencing, - Import.Stage.BlockFollowing + Import.Stage.BlockFollowing, + Import.Stage.BlockPending ] # in order so that foreign keys are inserted before being referenced diff --git a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex index 2a47892e3f..872e13ae7b 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex @@ -8,7 +8,8 @@ defmodule Explorer.Chain.Import.Runner.Blocks do import Ecto.Query, only: [from: 2, subquery: 1] alias Ecto.{Changeset, Multi, Repo} - alias Explorer.Chain.{Address, Block, Import, InternalTransaction, Log, Transaction} + + alias Explorer.Chain.{Address, Block, Import, Log, PendingBlockOperation, Transaction} alias Explorer.Chain.Block.Reward alias Explorer.Chain.Import.Runner alias Explorer.Chain.Import.Runner.Address.CurrentTokenBalances @@ -56,6 +57,9 @@ defmodule Explorer.Chain.Import.Runner.Blocks do # Note, needs to be executed after `lose_consensus` for lock acquisition insert(repo, changes_list, insert_options) end) + |> Multi.run(:new_pending_operations, fn repo, %{lose_consensus: nonconsensus_hashes} -> + new_pending_operations(repo, nonconsensus_hashes, hashes, insert_options) + end) |> Multi.run(:uncle_fetched_block_second_degree_relations, fn repo, _ -> update_block_second_degree_relations(repo, hashes, %{ timeout: @@ -86,9 +90,6 @@ defmodule Explorer.Chain.Import.Runner.Blocks do |> Multi.run(:remove_nonconsensus_logs, fn repo, %{derive_transaction_forks: transactions} -> remove_nonconsensus_logs(repo, transactions, insert_options) end) - |> Multi.run(:remove_nonconsensus_internal_transactions, fn repo, %{derive_transaction_forks: transactions} -> - remove_nonconsensus_internal_transactions(repo, transactions, insert_options) - end) |> Multi.run(:acquire_contract_address_tokens, fn repo, _ -> acquire_contract_address_tokens(repo, consensus_block_numbers) end) @@ -157,7 +158,6 @@ defmodule Explorer.Chain.Import.Runner.Blocks do gas_used: nil, cumulative_gas_used: nil, index: nil, - internal_transactions_indexed_at: nil, status: nil, error: nil, updated_at: ^updated_at @@ -248,7 +248,6 @@ defmodule Explorer.Chain.Import.Runner.Blocks do difficulty: fragment("EXCLUDED.difficulty"), gas_limit: fragment("EXCLUDED.gas_limit"), gas_used: fragment("EXCLUDED.gas_used"), - internal_transactions_indexed_at: fragment("EXCLUDED.internal_transactions_indexed_at"), miner_hash: fragment("EXCLUDED.miner_hash"), nonce: fragment("EXCLUDED.nonce"), number: fragment("EXCLUDED.number"), @@ -267,8 +266,7 @@ defmodule Explorer.Chain.Import.Runner.Blocks do fragment("EXCLUDED.miner_hash <> ?", block.miner_hash) or fragment("EXCLUDED.nonce <> ?", block.nonce) or fragment("EXCLUDED.number <> ?", block.number) or fragment("EXCLUDED.parent_hash <> ?", block.parent_hash) or fragment("EXCLUDED.size <> ?", block.size) or fragment("EXCLUDED.timestamp <> ?", block.timestamp) or - fragment("EXCLUDED.total_difficulty <> ?", block.total_difficulty) or - fragment("EXCLUDED.internal_transactions_indexed_at <> ?", block.internal_transactions_indexed_at) + fragment("EXCLUDED.total_difficulty <> ?", block.total_difficulty) ) end @@ -314,34 +312,27 @@ defmodule Explorer.Chain.Import.Runner.Blocks do {:error, %{exception: postgrex_error, consensus_block_numbers: consensus_block_numbers}} end - defp remove_nonconsensus_internal_transactions(repo, forked_transaction_hashes, %{timeout: timeout}) do - query = - from( - internal_transaction in InternalTransaction, - where: internal_transaction.transaction_hash in ^forked_transaction_hashes, - select: %{transaction_hash: internal_transaction.transaction_hash}, - # Enforce InternalTransaction ShareLocks order (see docs: sharelocks.md) - order_by: [ - internal_transaction.transaction_hash, - internal_transaction.index - ], - lock: "FOR UPDATE" - ) - - delete_query = - from( - i in InternalTransaction, - join: s in subquery(query), - on: i.transaction_hash == s.transaction_hash, - select: map(i, [:transaction_hash, :index]) - ) - - {_count, deleted_internal_transactions} = repo.delete_all(delete_query, timeout: timeout) + defp new_pending_operations(repo, nonconsensus_hashes, hashes, %{timeout: timeout, timestamps: timestamps}) do + sorted_pending_ops = + nonconsensus_hashes + |> MapSet.new() + |> MapSet.union(MapSet.new(hashes)) + |> MapSet.to_list() + |> Enum.sort() + |> Enum.map(fn hash -> + %{block_hash: hash, fetch_internal_transactions: true} + end) - {:ok, deleted_internal_transactions} - rescue - postgrex_error in Postgrex.Error -> - {:error, %{exception: postgrex_error, transactions: forked_transaction_hashes}} + Import.insert_changes_list( + repo, + sorted_pending_ops, + conflict_target: :block_hash, + on_conflict: PendingBlockOperation.default_on_conflict(), + for: PendingBlockOperation, + returning: true, + timeout: timeout, + timestamps: timestamps + ) end defp remove_nonconsensus_logs(repo, forked_transaction_hashes, %{timeout: timeout}) do diff --git a/apps/explorer/lib/explorer/chain/import/runner/internal_transactions.ex b/apps/explorer/lib/explorer/chain/import/runner/internal_transactions.ex index 9d16328555..38ec79b5a9 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/internal_transactions.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/internal_transactions.ex @@ -6,11 +6,12 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactions do require Ecto.Query require Logger + alias Ecto.Adapters.SQL alias Ecto.{Changeset, Multi, Repo} - alias Explorer.Chain.{Block, Hash, Import, InternalTransaction, Transaction} + alias Explorer.Chain.{Block, Hash, Import, InternalTransaction, PendingBlockOperation, Transaction} alias Explorer.Chain.Import.Runner - import Ecto.Query, only: [from: 2] + import Ecto.Query, only: [from: 2, or_where: 3] @behaviour Runner @@ -19,6 +20,7 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactions do @type imported :: [ %{required(:index) => non_neg_integer(), required(:transaction_hash) => Hash.Full.t()} + | %{required(:empty_block_number) => non_neg_integer()} ] @impl Runner @@ -48,54 +50,72 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactions do update_transactions_options = %{timeout: transactions_timeout, timestamps: timestamps} + internal_transactions_changes = Enum.filter(changes_list, &Map.has_key?(&1, :type)) + # Enforce ShareLocks tables order (see docs: sharelocks.md) multi - |> Multi.run(:acquire_transactions, fn repo, _ -> - acquire_transactions(repo, changes_list) + |> Multi.run(:acquire_blocks, fn repo, _ -> + acquire_blocks(repo, changes_list) end) - |> Multi.run(:internal_transactions, fn repo, %{acquire_transactions: transactions} -> - insert(repo, changes_list, transactions, insert_options) + |> Multi.run(:acquire_pending_internal_txs, fn repo, %{acquire_blocks: block_hashes} -> + acquire_pending_internal_txs(repo, block_hashes) end) - |> Multi.run(:internal_transactions_indexed_at_transactions, fn repo, %{acquire_transactions: transactions} -> - update_transactions(repo, transactions, update_transactions_options) + |> Multi.run(:acquire_transactions, fn repo, %{acquire_pending_internal_txs: pending_block_hashes} -> + acquire_transactions(repo, pending_block_hashes) + end) + |> Multi.run(:invalid_block_numbers, fn _, %{acquire_transactions: transactions} -> + invalid_block_numbers(transactions, internal_transactions_changes) + end) + |> Multi.run(:valid_internal_transactions, fn _, + %{ + acquire_transactions: transactions, + invalid_block_numbers: invalid_block_numbers + } -> + valid_internal_transactions(transactions, internal_transactions_changes, invalid_block_numbers) + end) + |> Multi.run(:internal_transactions, fn repo, %{valid_internal_transactions: valid_internal_transactions} -> + insert(repo, valid_internal_transactions, insert_options) + end) + |> Multi.run(:remove_left_over_internal_transactions, fn repo, + %{valid_internal_transactions: valid_internal_transactions} -> + remove_left_over_internal_transactions(repo, valid_internal_transactions) + end) + |> Multi.run(:update_transactions, fn repo, %{valid_internal_transactions: valid_internal_transactions} -> + update_transactions(repo, valid_internal_transactions, update_transactions_options) + end) + |> Multi.run(:remove_consensus_of_invalid_blocks, fn repo, %{invalid_block_numbers: invalid_block_numbers} -> + remove_consensus_of_invalid_blocks(repo, invalid_block_numbers) + end) + |> Multi.run(:update_pending_blocks_status, fn repo, + %{ + acquire_pending_internal_txs: pending_block_hashes, + remove_consensus_of_invalid_blocks: invalid_block_hashes + } -> + update_pending_blocks_status(repo, pending_block_hashes, invalid_block_hashes) end) - |> Multi.run( - :remove_consensus_of_missing_transactions_blocks, - fn repo, %{internal_transactions: inserted} = results_map -> - # NOTE: for this to work it has to follow the runner `internal_transactions_indexed_at_blocks` - block_hashes = Map.get(results_map, :internal_transactions_indexed_at_blocks, []) - remove_consensus_of_missing_transactions_blocks(repo, block_hashes, changes_list, inserted) - end - ) end @impl Runner def timeout, do: @timeout - @spec insert(Repo.t(), [map], [Transaction.t()], %{ + @spec insert(Repo.t(), [map], %{ optional(:on_conflict) => Runner.on_conflict(), required(:timeout) => timeout, required(:timestamps) => Import.timestamps() }) :: {:ok, [%{index: non_neg_integer, transaction_hash: Hash.t()}]} | {:error, [Changeset.t()]} - defp insert(repo, changes_list, transactions, %{timeout: timeout, timestamps: timestamps} = options) - when is_list(changes_list) do + defp insert(repo, valid_internal_transactions, %{timeout: timeout, timestamps: timestamps} = options) + when is_list(valid_internal_transactions) do on_conflict = Map.get_lazy(options, :on_conflict, &default_on_conflict/0) - transactions_map = Map.new(transactions, &{&1.hash, &1}) - - final_changes_list = - changes_list - # Enforce InternalTransaction ShareLocks order (see docs: sharelocks.md) - |> Enum.sort_by(&{&1.transaction_hash, &1.index}) - |> reject_missing_transactions(transactions_map) + ordered_changes_list = Enum.sort_by(valid_internal_transactions, &{&1.transaction_hash, &1.index}) {:ok, internal_transactions} = Import.insert_changes_list( repo, - final_changes_list, - conflict_target: [:transaction_hash, :index], + ordered_changes_list, + conflict_target: [:block_hash, :block_index], for: InternalTransaction, on_conflict: on_conflict, returning: true, @@ -119,24 +139,28 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactions do from_address_hash: fragment("EXCLUDED.from_address_hash"), gas: fragment("EXCLUDED.gas"), gas_used: fragment("EXCLUDED.gas_used"), - # Don't update `index` as it is part of the composite primary key and used for the conflict target + index: fragment("EXCLUDED.index"), init: fragment("EXCLUDED.init"), input: fragment("EXCLUDED.input"), output: fragment("EXCLUDED.output"), to_address_hash: fragment("EXCLUDED.to_address_hash"), trace_address: fragment("EXCLUDED.trace_address"), - # Don't update `transaction_hash` as it is part of the composite primary key and used for the conflict target + transaction_hash: fragment("EXCLUDED.transaction_hash"), transaction_index: fragment("EXCLUDED.transaction_index"), type: fragment("EXCLUDED.type"), value: fragment("EXCLUDED.value"), inserted_at: fragment("LEAST(?, EXCLUDED.inserted_at)", internal_transaction.inserted_at), updated_at: fragment("GREATEST(?, EXCLUDED.updated_at)", internal_transaction.updated_at) + # Don't update `block_hash` as it is used for the conflict target + # Don't update `block_index` as it is used for the conflict target ] ], # `IS DISTINCT FROM` is used because it allows `NULL` to be equal to itself where: fragment( - "(EXCLUDED.call_type, EXCLUDED.created_contract_address_hash, EXCLUDED.created_contract_code, EXCLUDED.error, EXCLUDED.from_address_hash, EXCLUDED.gas, EXCLUDED.gas_used, EXCLUDED.init, EXCLUDED.input, EXCLUDED.output, EXCLUDED.to_address_hash, EXCLUDED.trace_address, EXCLUDED.transaction_index, EXCLUDED.type, EXCLUDED.value) IS DISTINCT FROM (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + "(EXCLUDED.transaction_hash, EXCLUDED.index, EXCLUDED.call_type, EXCLUDED.created_contract_address_hash, EXCLUDED.created_contract_code, EXCLUDED.error, EXCLUDED.from_address_hash, EXCLUDED.gas, EXCLUDED.gas_used, EXCLUDED.init, EXCLUDED.input, EXCLUDED.output, EXCLUDED.to_address_hash, EXCLUDED.trace_address, EXCLUDED.transaction_index, EXCLUDED.type, EXCLUDED.value) IS DISTINCT FROM (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + internal_transaction.transaction_hash, + internal_transaction.index, internal_transaction.call_type, internal_transaction.created_contract_address_hash, internal_transaction.created_contract_code, @@ -156,18 +180,41 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactions do ) end - defp acquire_transactions(repo, internal_transactions) do - transaction_hashes = - internal_transactions - |> MapSet.new(& &1.transaction_hash) - |> MapSet.to_list() + defp acquire_blocks(repo, changes_list) do + block_numbers = Enum.map(changes_list, & &1.block_number) + + query = + from( + b in Block, + where: b.number in ^block_numbers and b.consensus, + select: b.hash, + # Enforce Block ShareLocks order (see docs: sharelocks.md) + order_by: [asc: b.hash], + lock: "FOR UPDATE" + ) + + {:ok, repo.all(query)} + end + defp acquire_pending_internal_txs(repo, block_hashes) do + query = + from( + pending_ops in PendingBlockOperation, + where: pending_ops.block_hash in ^block_hashes, + select: pending_ops.block_hash, + # Enforce PendingBlockOperation ShareLocks order (see docs: sharelocks.md) + order_by: [asc: pending_ops.block_hash], + lock: "FOR UPDATE" + ) + + {:ok, repo.all(query)} + end + + defp acquire_transactions(repo, pending_block_hashes) do query = from( t in Transaction, - where: t.hash in ^transaction_hashes, - # do not consider pending transactions - where: not is_nil(t.block_hash), + where: t.block_hash in ^pending_block_hashes, select: map(t, [:hash, :block_hash, :block_number]), # Enforce Transaction ShareLocks order (see docs: sharelocks.md) order_by: t.hash, @@ -177,22 +224,96 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactions do {:ok, repo.all(query)} end - defp update_transactions(repo, transactions, %{ + defp invalid_block_numbers(transactions, internal_transactions_changes) do + required_tuples = MapSet.new(transactions, &{&1.hash, &1.block_number}) + + candidate_tuples = MapSet.new(internal_transactions_changes, &{&1.transaction_hash, &1.block_number}) + + all_tuples = MapSet.union(required_tuples, candidate_tuples) + + common_tuples = MapSet.intersection(required_tuples, candidate_tuples) + + invalid_numbers = + all_tuples + |> MapSet.difference(common_tuples) + |> MapSet.new(fn {_hash, block_number} -> block_number end) + |> MapSet.to_list() + + {:ok, invalid_numbers} + end + + defp valid_internal_transactions(transactions, internal_transactions_changes, invalid_block_numbers) do + blocks_map = Map.new(transactions, &{&1.block_number, &1.block_hash}) + + valid_internal_txs = + internal_transactions_changes + |> Enum.group_by(& &1.block_number) + |> Map.drop(invalid_block_numbers) + |> Enum.flat_map(fn {block_number, entries} -> + block_hash = Map.fetch!(blocks_map, block_number) + + entries + |> Enum.sort_by(&{&1.transaction_hash, &1.index}) + |> Enum.with_index() + |> Enum.map(fn {entry, index} -> + entry + |> Map.put(:block_hash, block_hash) + |> Map.put(:block_index, index) + end) + end) + + {:ok, valid_internal_txs} + end + + def defer_internal_transactions_primary_key(repo) do + # allows internal_transactions primary key to skips being checked during the + # DB transactions and instead be checked at the end of it. + # This allows us to use a more efficient upserting logic + SQL.query(repo, "SET CONSTRAINTS internal_transactions_pkey DEFERRED") + end + + def remove_left_over_internal_transactions(repo, valid_internal_transactions) do + case valid_internal_transactions do + [] -> + {:ok, []} + + _ -> + try do + delete_query = + valid_internal_transactions + |> Enum.group_by(& &1.block_hash, & &1.block_index) + |> Enum.map(fn {block_hash, indexes} -> {block_hash, Enum.max(indexes)} end) + |> Enum.reduce(InternalTransaction, fn {block_hash, max_index}, acc -> + or_where(acc, [it], it.block_hash == ^block_hash and it.block_index > ^max_index) + end) + + # ShareLocks order already enforced by `acquire_pending_internal_txs` (see docs: sharelocks.md) + {_count, result} = repo.delete_all(delete_query, []) + + {:ok, result} + rescue + postgrex_error in Postgrex.Error -> {:error, %{exception: postgrex_error}} + end + end + end + + defp update_transactions(repo, valid_internal_transactions, %{ timeout: timeout, timestamps: timestamps }) - when is_list(transactions) do - transaction_hashes = Enum.map(transactions, & &1.hash) + when is_list(valid_internal_transactions) do + transaction_hashes = + valid_internal_transactions + |> MapSet.new(& &1.transaction_hash) + |> MapSet.to_list() update_query = from( t in Transaction, - # pending transactions are already excluded by `acquire_transactions` where: t.hash in ^transaction_hashes, # ShareLocks order already enforced by `acquire_transactions` (see docs: sharelocks.md) update: [ set: [ - internal_transactions_indexed_at: ^timestamps.updated_at, created_contract_address_hash: fragment( "(SELECT it.created_contract_address_hash FROM internal_transactions AS it WHERE it.transaction_hash = ? ORDER BY it.index ASC LIMIT 1)", @@ -209,7 +330,8 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactions do t.hash, type(^:ok, t.status), type(^:error, t.status) - ) + ), + updated_at: ^timestamps.updated_at ] ] ) @@ -224,26 +346,14 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactions do end end - # If not using Parity this is not relevant - defp remove_consensus_of_missing_transactions_blocks(_, [], _, _), do: {:ok, []} - - defp remove_consensus_of_missing_transactions_blocks(repo, block_hashes, changes_list, inserted) do - inserted_block_numbers = MapSet.new(inserted, & &1.block_number) - - missing_transactions_block_numbers = - changes_list - |> MapSet.new(& &1.block_number) - |> MapSet.difference(inserted_block_numbers) - |> MapSet.to_list() - + defp remove_consensus_of_invalid_blocks(repo, invalid_block_numbers) do update_query = from( b in Block, - where: b.number in ^missing_transactions_block_numbers, - where: b.hash in ^block_hashes, - select: b.number, - # ShareLocks order already enforced by `internal_transactions_indexed_at_blocks` (see docs: sharelocks.md) - update: [set: [consensus: false, internal_transactions_indexed_at: nil]] + where: b.number in ^invalid_block_numbers and b.consensus, + select: b.hash, + # ShareLocks order already enforced by `acquire_blocks` (see docs: sharelocks.md) + update: [set: [consensus: false]] ) try do @@ -252,24 +362,39 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactions do Logger.debug(fn -> [ "consensus removed from blocks with numbers: ", - inspect(missing_transactions_block_numbers), - " because of missing transactions" + inspect(invalid_block_numbers), + " because of mismatching transactions" ] end) {:ok, result} rescue postgrex_error in Postgrex.Error -> - {:error, %{exception: postgrex_error, missing_transactions_block_numbers: missing_transactions_block_numbers}} + {:error, %{exception: postgrex_error, invalid_block_numbers: invalid_block_numbers}} end end - defp reject_missing_transactions(ordered_changes_list, transactions_map) do - Enum.reject(ordered_changes_list, fn %{transaction_hash: hash} -> - transactions_map - |> Map.get(hash, %{}) - |> Map.get(:block_hash) - |> is_nil() - end) + def update_pending_blocks_status(repo, pending_hashes, invalid_block_hashes) do + valid_block_hashes = + pending_hashes + |> MapSet.new() + |> MapSet.difference(MapSet.new(invalid_block_hashes)) + |> MapSet.to_list() + + delete_query = + from( + pending_ops in PendingBlockOperation, + where: pending_ops.block_hash in ^valid_block_hashes + ) + + try do + # ShreLocks order already enforced by `acquire_pending_internal_txs` (see docs: sharelocks.md) + {_count, deleted} = repo.delete_all(delete_query, []) + + {:ok, deleted} + rescue + postgrex_error in Postgrex.Error -> + {:error, %{exception: postgrex_error, pending_hashes: valid_block_hashes}} + end end end diff --git a/apps/explorer/lib/explorer/chain/import/runner/internal_transactions_indexed_at_blocks.ex b/apps/explorer/lib/explorer/chain/import/runner/internal_transactions_indexed_at_blocks.ex deleted file mode 100644 index 048a94942c..0000000000 --- a/apps/explorer/lib/explorer/chain/import/runner/internal_transactions_indexed_at_blocks.ex +++ /dev/null @@ -1,82 +0,0 @@ -defmodule Explorer.Chain.Import.Runner.InternalTransactionsIndexedAtBlocks do - @moduledoc """ - Bulk updates `internal_transactions_indexed_at` for provided blocks - """ - - require Ecto.Query - - alias Ecto.Multi - alias Explorer.Chain.Block - alias Explorer.Chain.Import.Runner - - import Ecto.Query, only: [from: 2] - - @behaviour Runner - - # milliseconds - @timeout 60_000 - - @type imported :: [%{number: Block.block_number()}] - - @impl Runner - def ecto_schema_module, do: Block - - @impl Runner - def option_key, do: :internal_transactions_indexed_at_blocks - - @impl Runner - def imported_table_row do - %{ - value_type: "[%{number: Explorer.Chain.Block.block_number()}]", - value_description: "List of block numbers to set `internal_transactions_indexed_at` field for" - } - end - - @impl Runner - def run(multi, changes_list, %{timestamps: timestamps} = options) when is_map(options) do - transactions_timeout = options[Runner.Transactions.option_key()][:timeout] || Runner.Transactions.timeout() - - update_transactions_options = %{timeout: transactions_timeout, timestamps: timestamps} - - multi - |> Multi.run(:internal_transactions_indexed_at_blocks, fn repo, _ -> - update_blocks(repo, changes_list, update_transactions_options) - end) - end - - @impl Runner - def timeout, do: @timeout - - defp update_blocks(_repo, [], %{}), do: {:ok, []} - - defp update_blocks(repo, changes_list, %{ - timeout: timeout, - timestamps: timestamps - }) - when is_list(changes_list) do - block_numbers = Enum.map(changes_list, fn %{number: number} -> number end) - - query = - from( - b in Block, - where: b.number in ^block_numbers and b.consensus, - # Enforce Block ShareLocks order (see docs: sharelocks.md) - order_by: [asc: b.hash], - lock: "FOR UPDATE" - ) - - try do - {_, result} = - repo.update_all( - from(b in Block, join: s in subquery(query), on: b.hash == s.hash, select: b.hash), - [set: [internal_transactions_indexed_at: timestamps.updated_at]], - timeout: timeout - ) - - {:ok, result} - rescue - postgrex_error in Postgrex.Error -> - {:error, %{exception: postgrex_error, block_numbers: block_numbers}} - end - end -end diff --git a/apps/explorer/lib/explorer/chain/import/runner/transactions.ex b/apps/explorer/lib/explorer/chain/import/runner/transactions.ex index dcc27ca0da..7a7265d1ae 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/transactions.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/transactions.ex @@ -8,7 +8,7 @@ defmodule Explorer.Chain.Import.Runner.Transactions do import Ecto.Query, only: [from: 2] alias Ecto.{Multi, Repo} - alias Explorer.Chain.{Block, Data, Hash, Import, Transaction} + alias Explorer.Chain.{Block, Hash, Import, Transaction} alias Explorer.Chain.Import.Runner.TokenTransfers @behaviour Import.Runner @@ -72,8 +72,7 @@ defmodule Explorer.Chain.Import.Runner.Transactions do changes_list, %{ timeout: timeout, - timestamps: %{inserted_at: inserted_at} = timestamps, - token_transfer_transaction_hash_set: token_transfer_transaction_hash_set + timestamps: timestamps } = options ) when is_list(changes_list) do @@ -81,7 +80,6 @@ defmodule Explorer.Chain.Import.Runner.Transactions do ordered_changes_list = changes_list - |> put_internal_transactions_indexed_at(inserted_at, token_transfer_transaction_hash_set) # Enforce Transaction ShareLocks order (see docs: sharelocks.md) |> Enum.sort_by(& &1.hash) @@ -114,7 +112,6 @@ defmodule Explorer.Chain.Import.Runner.Transactions do gas_price: fragment("EXCLUDED.gas_price"), gas_used: fragment("EXCLUDED.gas_used"), index: fragment("EXCLUDED.index"), - internal_transactions_indexed_at: fragment("EXCLUDED.internal_transactions_indexed_at"), input: fragment("EXCLUDED.input"), nonce: fragment("EXCLUDED.nonce"), r: fragment("EXCLUDED.r"), @@ -130,7 +127,7 @@ defmodule Explorer.Chain.Import.Runner.Transactions do ], where: fragment( - "(EXCLUDED.block_hash, EXCLUDED.block_number, EXCLUDED.created_contract_address_hash, EXCLUDED.created_contract_code_indexed_at, EXCLUDED.cumulative_gas_used, EXCLUDED.cumulative_gas_used, EXCLUDED.from_address_hash, EXCLUDED.gas, EXCLUDED.gas_price, EXCLUDED.gas_used, EXCLUDED.index, EXCLUDED.internal_transactions_indexed_at, EXCLUDED.input, EXCLUDED.nonce, EXCLUDED.r, EXCLUDED.s, EXCLUDED.status, EXCLUDED.to_address_hash, EXCLUDED.v, EXCLUDED.value) IS DISTINCT FROM (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + "(EXCLUDED.block_hash, EXCLUDED.block_number, EXCLUDED.created_contract_address_hash, EXCLUDED.created_contract_code_indexed_at, EXCLUDED.cumulative_gas_used, EXCLUDED.cumulative_gas_used, EXCLUDED.from_address_hash, EXCLUDED.gas, EXCLUDED.gas_price, EXCLUDED.gas_used, EXCLUDED.index, EXCLUDED.input, EXCLUDED.nonce, EXCLUDED.r, EXCLUDED.s, EXCLUDED.status, EXCLUDED.to_address_hash, EXCLUDED.v, EXCLUDED.value) IS DISTINCT FROM (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", transaction.block_hash, transaction.block_number, transaction.created_contract_address_hash, @@ -142,7 +139,6 @@ defmodule Explorer.Chain.Import.Runner.Transactions do transaction.gas_price, transaction.gas_used, transaction.index, - transaction.internal_transactions_indexed_at, transaction.input, transaction.nonce, transaction.r, @@ -155,46 +151,6 @@ defmodule Explorer.Chain.Import.Runner.Transactions do ) end - defp put_internal_transactions_indexed_at(changes_list, timestamp, token_transfer_transaction_hash_set) do - if Application.get_env(:explorer, :index_internal_transactions_for_token_transfers) do - changes_list - else - do_put_internal_transactions_indexed_at(changes_list, timestamp, token_transfer_transaction_hash_set) - end - end - - defp do_put_internal_transactions_indexed_at(changes_list, timestamp, token_transfer_transaction_hash_set) - when is_list(changes_list) do - Enum.map(changes_list, &put_internal_transactions_indexed_at(&1, timestamp, token_transfer_transaction_hash_set)) - end - - defp do_put_internal_transactions_indexed_at(%{hash: hash} = changes, timestamp, token_transfer_transaction_hash_set) do - token_transfer? = to_string(hash) in token_transfer_transaction_hash_set - - if put_internal_transactions_indexed_at?(changes, token_transfer?) do - Map.put(changes, :internal_transactions_indexed_at, timestamp) - else - changes - end - end - - # A post-Byzantium validated transaction will have a status and if it has no input, it is a value transfer only. - # Internal transactions are only needed when status is `:error` to set `error`. - defp put_internal_transactions_indexed_at?(%{status: :ok, input: %Data{bytes: <<>>}}, _), do: true - - # A post-Byzantium validated transaction will have a status and if it transfers tokens, the token transfer is in the - # log and the internal transactions. - # `created_contract_address_hash` must be `nil` because if a contract is created the internal transactions are needed - # to get - defp put_internal_transactions_indexed_at?(%{status: :ok} = changes, true) do - case Map.fetch(changes, :created_contract_address_hash) do - {:ok, created_contract_address_hash} when not is_nil(created_contract_address_hash) -> false - :error -> true - end - end - - defp put_internal_transactions_indexed_at?(_, _), do: false - defp discard_blocks_for_recollated_transactions(repo, changes_list, %{ timeout: timeout, timestamps: %{updated_at: updated_at} diff --git a/apps/explorer/lib/explorer/chain/import/stage/block_following.ex b/apps/explorer/lib/explorer/chain/import/stage/block_following.ex index 9b64fca993..65a7c07603 100644 --- a/apps/explorer/lib/explorer/chain/import/stage/block_following.ex +++ b/apps/explorer/lib/explorer/chain/import/stage/block_following.ex @@ -13,10 +13,8 @@ defmodule Explorer.Chain.Import.Stage.BlockFollowing do @impl Stage def runners, do: [ - Runner.InternalTransactionsIndexedAtBlocks, Runner.Block.SecondDegreeRelations, Runner.Block.Rewards, - Runner.InternalTransactions, Runner.Address.CurrentTokenBalances ] diff --git a/apps/explorer/lib/explorer/chain/import/stage/block_pending.ex b/apps/explorer/lib/explorer/chain/import/stage/block_pending.ex new file mode 100644 index 0000000000..ef8088296d --- /dev/null +++ b/apps/explorer/lib/explorer/chain/import/stage/block_pending.ex @@ -0,0 +1,26 @@ +defmodule Explorer.Chain.Import.Stage.BlockPending do + @moduledoc """ + Imports any tables that follows and cannot be imported at the same time as + those imported by `Explorer.Chain.Import.Stage.Addresses`, + `Explorer.Chain.Import.Stage.AddressReferencing` and + `Explorer.Chain.Import.Stage.BlockReferencing` + """ + + alias Explorer.Chain.Import.{Runner, Stage} + + @behaviour Stage + + @impl Stage + def runners, + do: [ + Runner.InternalTransactions + ] + + @impl Stage + def multis(runner_to_changes_list, options) do + {final_multi, final_remaining_runner_to_changes_list} = + Stage.single_multi(runners(), runner_to_changes_list, options) + + {[final_multi], final_remaining_runner_to_changes_list} + end +end diff --git a/apps/explorer/lib/explorer/chain/internal_transaction.ex b/apps/explorer/lib/explorer/chain/internal_transaction.ex index 77f7d15da2..cd6c7996fd 100644 --- a/apps/explorer/lib/explorer/chain/internal_transaction.ex +++ b/apps/explorer/lib/explorer/chain/internal_transaction.ex @@ -3,7 +3,7 @@ defmodule Explorer.Chain.InternalTransaction do use Explorer.Schema - alias Explorer.Chain.{Address, Data, Gas, Hash, Transaction, Wei} + alias Explorer.Chain.{Address, Block, Data, Gas, Hash, PendingBlockOperation, Transaction, Wei} alias Explorer.Chain.InternalTransaction.{Action, CallType, Result, Type} @typedoc """ @@ -50,7 +50,9 @@ defmodule Explorer.Chain.InternalTransaction do transaction: %Ecto.Association.NotLoaded{} | Transaction.t(), transaction_hash: Hash.t(), transaction_index: Transaction.transaction_index() | nil, - value: Wei.t() + value: Wei.t(), + block_hash: Hash.Full.t(), + block_index: non_neg_integer() } @primary_key false @@ -69,6 +71,7 @@ defmodule Explorer.Chain.InternalTransaction do field(:value, Wei) field(:block_number, :integer) field(:transaction_index, :integer) + field(:block_index, :integer) timestamps() @@ -102,6 +105,20 @@ defmodule Explorer.Chain.InternalTransaction do references: :hash, type: Hash.Full ) + + belongs_to(:block, Block, + foreign_key: :block_hash, + references: :hash, + type: Hash.Full + ) + + belongs_to(:pending_block, PendingBlockOperation, + foreign_key: :block_hash, + define_field: false, + references: :block_hash, + type: Hash.Full, + where: [fetch_internal_transactions: true] + ) end @doc """ @@ -125,7 +142,9 @@ defmodule Explorer.Chain.InternalTransaction do ...> transaction_hash: "0x3a3eb134e6792ce9403ea4188e5e79693de9e4c94e499db132be086400da79e6", ...> type: "create", ...> value: 0, - ...> block_number: 35 + ...> block_number: 35, + ...> block_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd", + ...> block_index: 0 ...> } ...> ) iex> changeset.valid? @@ -165,6 +184,8 @@ defmodule Explorer.Chain.InternalTransaction do ...> type: "create", ...> value: 0, ...> block_number: 35, + ...> block_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd", + ...> block_index: 0, ...> transaction_index: 0 ...> } iex> ) @@ -182,6 +203,8 @@ defmodule Explorer.Chain.InternalTransaction do ...> %Explorer.Chain.InternalTransaction{}, ...> %{ ...> block_number: 35, + ...> block_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd", + ...> block_index: 0, ...> transaction_index: 0, ...> transaction_hash: "0x3a3eb134e6792ce9403ea4188e5e79693de9e4c94e499db132be086400da79e6", ...> index: 0, @@ -206,6 +229,8 @@ defmodule Explorer.Chain.InternalTransaction do ...> %Explorer.Chain.InternalTransaction{}, ...> %{ ...> block_number: 35, + ...> block_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd", + ...> block_index: 0, ...> transaction_index: 0, ...> transaction_hash: "0xcd7c15dbbc797722bef6e1d551edfd644fc7f4fb2ccd6a7947b2d1ade9ed140b", ...> index: 0, @@ -230,6 +255,8 @@ defmodule Explorer.Chain.InternalTransaction do ...> %Explorer.Chain.InternalTransaction{}, ...> %{ ...> block_number: 35, + ...> block_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd", + ...> block_index: 0, ...> transaction_index: 0, ...> transaction_hash: "0xcd7c15dbbc797722bef6e1d551edfd644fc7f4fb2ccd6a7947b2d1ade9ed140b", ...> index: 0, @@ -260,6 +287,8 @@ defmodule Explorer.Chain.InternalTransaction do ...> %Explorer.Chain.InternalTransaction{}, ...> %{ ...> block_number: 35, + ...> block_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd", + ...> block_index: 0, ...> transaction_index: 0, ...> transaction_hash: "0xcd7c15dbbc797722bef6e1d551edfd644fc7f4fb2ccd6a7947b2d1ade9ed140b", ...> index: 0, @@ -300,6 +329,8 @@ defmodule Explorer.Chain.InternalTransaction do ...> type: "create", ...> value: 0, ...> block_number: 35, + ...> block_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd", + ...> block_index: 0, ...> transaction_index: 0 ...> } iex> ) @@ -326,6 +357,8 @@ defmodule Explorer.Chain.InternalTransaction do ...> type: "create", ...> value: 0, ...> block_number: 35, + ...> block_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd", + ...> block_index: 0, ...> transaction_index: 0 ...> } ...> ) @@ -351,6 +384,8 @@ defmodule Explorer.Chain.InternalTransaction do ...> type: "selfdestruct", ...> value: 0, ...> block_number: 35, + ...> block_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd", + ...> block_index: 0, ...> transaction_index: 0 ...> } ...> ) @@ -362,9 +397,39 @@ defmodule Explorer.Chain.InternalTransaction do internal_transaction |> cast(attrs, ~w(type)a) |> validate_required(~w(type)a) + |> validate_block_required(attrs) |> type_changeset(attrs) end + @doc """ + Accepts changes without `:type` but with `:block_number`, if `:type` is defined + works like `changeset`, except allowing `:block_hash` and `:block_index` to be undefined. + + This is used because the `internal_transactions` runner can derive such values + on its own or use empty types to know that a block has no internal transactions. + """ + def blockless_changeset(%__MODULE__{} = internal_transaction, attrs \\ %{}) do + changeset = + internal_transaction + |> cast(attrs, ~w(type block_number)a) + + if is_nil(get_field(changeset, :type)) do + changeset + |> validate_required(~w(block_number)a) + else + changeset + |> validate_required(~w(type)a) + |> type_changeset(attrs) + end + end + + defp validate_block_required(changeset, attrs) do + changeset + |> cast(attrs, ~w(block_hash block_index)a) + |> validate_required(~w(block_hash block_index)a) + |> foreign_key_constraint(:block_hash) + end + defp type_changeset(changeset, attrs) do type = get_field(changeset, :type) @@ -503,6 +568,16 @@ defmodule Explorer.Chain.InternalTransaction do where(query, [t], not is_nil(t.block_number)) end + @doc """ + Filters out internal_transactions of blocks that are flagged as needing fethching + of internal_transactions + """ + def where_nonpending_block(query \\ nil) do + (query || __MODULE__) + |> join(:left, [it], pending in assoc(it, :pending_block), as: :pending) + |> where([it, pending: pending], is_nil(pending.block_hash)) + end + def internal_transactions_to_raw(internal_transactions) when is_list(internal_transactions) do internal_transactions |> Enum.map(&internal_transaction_to_raw/1) diff --git a/apps/explorer/lib/explorer/chain/pending_block_operation.ex b/apps/explorer/lib/explorer/chain/pending_block_operation.ex index fa6ad4c2a5..d85cc589b0 100644 --- a/apps/explorer/lib/explorer/chain/pending_block_operation.ex +++ b/apps/explorer/lib/explorer/chain/pending_block_operation.ex @@ -7,17 +7,21 @@ defmodule Explorer.Chain.PendingBlockOperation do alias Explorer.Chain.{Block, Hash} - @required_attrs ~w(block_hash)a + @required_attrs ~w(block_hash fetch_internal_transactions)a @typedoc """ * `block_hash` - the hash of the block that has pending operations. + * `fetch_internal_transactions` - if the block needs its internal transactions fetched (or not) """ @type t :: %__MODULE__{ - block_hash: Hash.Full.t() + block_hash: Hash.Full.t(), + fetch_internal_transactions: boolean() } @primary_key false schema "pending_block_operations" do + field(:fetch_internal_transactions, :boolean) + timestamps() belongs_to(:block, Block, foreign_key: :block_hash, primary_key: true, references: :hash, type: Hash.Full) @@ -43,4 +47,41 @@ defmodule Explorer.Chain.PendingBlockOperation do lock: "FOR UPDATE" ) end + + def block_hashes(filter \\ nil) + + def block_hashes(filter) when is_nil(filter) do + from( + pending_ops in __MODULE__, + select: pending_ops.block_hash + ) + end + + def block_hashes(filters) when is_list(filters) do + true_filters = Keyword.new(filters, &{&1, true}) + + from( + pending_ops in __MODULE__, + where: ^true_filters, + select: pending_ops.block_hash + ) + end + + def block_hashes(filter), do: block_hashes([filter]) + + def default_on_conflict do + from( + pending_ops in __MODULE__, + update: [ + set: [ + fetch_internal_transactions: + pending_ops.fetch_internal_transactions or fragment("EXCLUDED.fetch_internal_transactions"), + # Don't update `block_hash` as it is used for the conflict target + inserted_at: pending_ops.inserted_at, + updated_at: fragment("EXCLUDED.updated_at") + ] + ], + where: fragment("EXCLUDED.fetch_internal_transactions <> ?", pending_ops.fetch_internal_transactions) + ) + end end diff --git a/apps/explorer/lib/explorer/chain/transaction.ex b/apps/explorer/lib/explorer/chain/transaction.ex index a17815514d..14d37d5c86 100644 --- a/apps/explorer/lib/explorer/chain/transaction.ex +++ b/apps/explorer/lib/explorer/chain/transaction.ex @@ -29,7 +29,7 @@ defmodule Explorer.Chain.Transaction do alias Explorer.Repo @optional_attrs ~w(block_hash block_number created_contract_address_hash cumulative_gas_used earliest_processing_start - error gas_used index internal_transactions_indexed_at created_contract_code_indexed_at status + error gas_used index created_contract_code_indexed_at status to_address_hash)a @required_attrs ~w(from_address_hash gas gas_price hash input nonce r s v value)a @@ -105,8 +105,6 @@ defmodule Explorer.Chain.Transaction do * `input`- data sent along with the transaction * `internal_transactions` - transactions (value transfers) created while executing contract used for this transaction - * `internal_transactions_indexed_at` - when `internal_transactions` were fetched by `Indexer` or when they do not - need to be fetched at `inserted_at`. * `created_contract_code_indexed_at` - when created `address` code was fetched by `Indexer` | `status` | `contract_creation_address_hash` | `input` | Token Transfer? | `internal_transactions_indexed_at` | `internal_transactions` | Description | @@ -152,7 +150,6 @@ defmodule Explorer.Chain.Transaction do index: transaction_index | nil, input: Data.t(), internal_transactions: %Ecto.Association.NotLoaded{} | [InternalTransaction.t()], - internal_transactions_indexed_at: DateTime.t(), logs: %Ecto.Association.NotLoaded{} | [Log.t()], nonce: non_neg_integer(), r: r(), @@ -174,7 +171,6 @@ defmodule Explorer.Chain.Transaction do :gas_price, :gas_used, :index, - :internal_transactions_indexed_at, :created_contract_code_indexed_at, :input, :nonce, @@ -195,7 +191,6 @@ defmodule Explorer.Chain.Transaction do field(:gas_price, Wei) field(:gas_used, :decimal) field(:index, :integer) - field(:internal_transactions_indexed_at, :utc_datetime_usec) field(:created_contract_code_indexed_at, :utc_datetime_usec) field(:input, Data) field(:nonce, :integer) diff --git a/apps/explorer/lib/explorer/etherscan.ex b/apps/explorer/lib/explorer/etherscan.ex index 41d8df9cb3..1a7e031376 100644 --- a/apps/explorer/lib/explorer/etherscan.ex +++ b/apps/explorer/lib/explorer/etherscan.ex @@ -97,6 +97,7 @@ defmodule Explorer.Etherscan do query |> Chain.where_transaction_has_multiple_internal_transactions() + |> InternalTransaction.where_nonpending_block() |> Repo.all() end @@ -140,6 +141,7 @@ defmodule Explorer.Etherscan do |> where_address_match(address_hash, options) |> where_start_block_match(options) |> where_end_block_match(options) + |> InternalTransaction.where_nonpending_block() |> Repo.all() end diff --git a/apps/explorer/lib/explorer/etherscan/logs.ex b/apps/explorer/lib/explorer/etherscan/logs.ex index 336eae106b..31e7d87c4e 100644 --- a/apps/explorer/lib/explorer/etherscan/logs.ex +++ b/apps/explorer/lib/explorer/etherscan/logs.ex @@ -79,7 +79,7 @@ defmodule Explorer.Etherscan.Logs do logs_query = where_topic_match(Log, prepared_filter) internal_transaction_log_query = - from(internal_transaction in InternalTransaction, + from(internal_transaction in InternalTransaction.where_nonpending_block(), join: transaction in assoc(internal_transaction, :transaction), join: log in ^logs_query, on: log.transaction_hash == internal_transaction.transaction_hash, diff --git a/apps/explorer/lib/explorer/graphql.ex b/apps/explorer/lib/explorer/graphql.ex index 5f1404a14f..a52a5beec4 100644 --- a/apps/explorer/lib/explorer/graphql.ex +++ b/apps/explorer/lib/explorer/graphql.ex @@ -40,7 +40,7 @@ defmodule Explorer.GraphQL do """ @spec get_internal_transaction(map()) :: {:ok, InternalTransaction.t()} | {:error, String.t()} def get_internal_transaction(%{transaction_hash: _, index: _} = clauses) do - if internal_transaction = Repo.get_by(InternalTransaction, clauses) do + if internal_transaction = Repo.get_by(InternalTransaction.where_nonpending_block(), clauses) do {:ok, internal_transaction} else {:error, "Internal transaction not found."} @@ -65,7 +65,9 @@ defmodule Explorer.GraphQL do select: it ) - Chain.where_transaction_has_multiple_internal_transactions(query) + query + |> InternalTransaction.where_nonpending_block() + |> Chain.where_transaction_has_multiple_internal_transactions() end @doc """ diff --git a/apps/explorer/priv/repo/migrations/20191018140054_add_pending_internal_txs_operation.exs b/apps/explorer/priv/repo/migrations/20191018140054_add_pending_internal_txs_operation.exs new file mode 100644 index 0000000000..73493e832c --- /dev/null +++ b/apps/explorer/priv/repo/migrations/20191018140054_add_pending_internal_txs_operation.exs @@ -0,0 +1,76 @@ +defmodule Explorer.Repo.Migrations.AddPendingInternalTxsOperation do + use Ecto.Migration + + def change do + alter table(:pending_block_operations) do + add(:fetch_internal_transactions, :boolean, null: false) + end + + execute(""" + INSERT INTO pending_block_operations + (block_hash, inserted_at, updated_at, fetch_internal_transactions) + SELECT b.hash, now(), now(), TRUE FROM blocks b + WHERE b.internal_transactions_indexed_at IS NULL + AND EXISTS ( + SELECT 1 + FROM transactions t + WHERE b.hash = t.block_hash + AND t.internal_transactions_indexed_at IS NULL + ); + """) + + alter table(:blocks) do + remove(:internal_transactions_indexed_at) + end + + alter table(:transactions) do + remove(:internal_transactions_indexed_at) + end + + alter table(:internal_transactions) do + add(:block_hash, :bytea) + add(:block_index, :integer) + end + + execute(""" + UPDATE internal_transactions itx + SET block_hash = with_block.block_hash, block_index = with_block.block_index + FROM ( + SELECT i.transaction_hash, + i.index, + t.block_hash, + row_number() OVER( + PARTITION BY t.block_hash + ORDER BY i.transaction_hash, i.index + ) - 1 AS block_index + FROM internal_transactions i + JOIN transactions t + ON t.hash = i.transaction_hash + ) AS with_block + WHERE itx.transaction_hash = with_block.transaction_hash + AND itx.index = with_block.index + ; + """) + + create(unique_index(:internal_transactions, [:block_hash, :block_index])) + + execute(""" + ALTER table internal_transactions + DROP CONSTRAINT internal_transactions_pkey, + ADD PRIMARY KEY (transaction_hash, index) DEFERRABLE; + """) + + alter table(:internal_transactions) do + modify(:block_hash, references(:blocks, column: :hash, type: :bytea), null: false) + modify(:block_index, :integer, null: false) + end + + drop( + index( + :internal_transactions, + [:transaction_hash, :index], + name: :internal_transactions_transaction_hash_index_index + ) + ) + end +end diff --git a/apps/explorer/test/explorer/chain/cache/pending_transactions_test.exs b/apps/explorer/test/explorer/chain/cache/pending_transactions_test.exs index 26339bc3d7..0d5a1e1ea5 100644 --- a/apps/explorer/test/explorer/chain/cache/pending_transactions_test.exs +++ b/apps/explorer/test/explorer/chain/cache/pending_transactions_test.exs @@ -9,9 +9,9 @@ defmodule Explorer.Chain.Cache.PendingTransactionsTest do PendingTransactions.update([transaction]) - transaction_hash = transaction.hash + assert [%{hash: pending_transaction_hash}] = PendingTransactions.all() - assert [%{hash: transaction_hash}] = PendingTransactions.all() + assert transaction.hash == pending_transaction_hash end end end diff --git a/apps/explorer/test/explorer/chain/cache/uncles_test.exs b/apps/explorer/test/explorer/chain/cache/uncles_test.exs index c8984d34ea..230889ca77 100644 --- a/apps/explorer/test/explorer/chain/cache/uncles_test.exs +++ b/apps/explorer/test/explorer/chain/cache/uncles_test.exs @@ -2,7 +2,6 @@ defmodule Explorer.Chain.Cache.UnclesTest do use Explorer.DataCase alias Explorer.Chain.Cache.Uncles - alias Explorer.Repo setup do Supervisor.terminate_child(Explorer.Supervisor, Uncles.child_id()) diff --git a/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs b/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs index 5eaaca5344..d6c278c1c8 100644 --- a/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs +++ b/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs @@ -7,7 +7,7 @@ defmodule Explorer.Chain.Import.Runner.BlocksTest do alias Ecto.Multi alias Explorer.Chain.Import.Runner.{Blocks, Transactions} - alias Explorer.Chain.{Address, Block, InternalTransaction, Log, Transaction, TokenTransfer} + alias Explorer.Chain.{Address, Block, Log, Transaction, TokenTransfer} alias Explorer.{Chain, Repo} describe "run/1" do @@ -131,24 +131,6 @@ defmodule Explorer.Chain.Import.Runner.BlocksTest do assert count(Log) == 0 end - test "remove_nonconsensus_internal_transactions deletes nonconsensus internal transactions", %{ - consensus_block: %{number: block_number} = block, - options: options - } do - old_block = insert(:block, number: block_number, consensus: true) - forked_transaction = :transaction |> insert() |> with_block(old_block) - - %InternalTransaction{index: index, transaction_hash: hash} = - insert(:internal_transaction, index: 0, transaction: forked_transaction) - - assert count(InternalTransaction) == 1 - - assert {:ok, %{remove_nonconsensus_internal_transactions: [%{transaction_hash: ^hash, index: ^index}]}} = - run_block_consensus_change(block, true, options) - - assert count(InternalTransaction) == 0 - end - test "derive_address_current_token_balances inserts rows if there is an address_token_balance left for the rows deleted by delete_address_current_token_balances", %{consensus_block: %{number: block_number} = block, options: options} do token = insert(:token) diff --git a/apps/explorer/test/explorer/chain/import/runner/internal_transactions_test.exs b/apps/explorer/test/explorer/chain/import/runner/internal_transactions_test.exs index 700106ffb5..96d54ca194 100644 --- a/apps/explorer/test/explorer/chain/import/runner/internal_transactions_test.exs +++ b/apps/explorer/test/explorer/chain/import/runner/internal_transactions_test.exs @@ -2,19 +2,20 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactionsTest do use Explorer.DataCase alias Ecto.Multi - alias Explorer.Chain.{Block, Data, Wei, Transaction, InternalTransaction} + alias Explorer.Chain.{Block, Data, Wei, PendingBlockOperation, Transaction, InternalTransaction} alias Explorer.Chain.Import.Runner.InternalTransactions describe "run/1" do test "transaction's status becomes :error when its internal_transaction has an error" do transaction = insert(:transaction) |> with_block(status: :ok) + insert(:pending_block_operation, block_hash: transaction.block_hash, fetch_internal_transactions: true) assert :ok == transaction.status index = 0 error = "Reverted" - internal_transaction_changes = make_internal_transaction_changes(transaction.hash, index, error) + internal_transaction_changes = make_internal_transaction_changes(transaction, index, error) assert {:ok, _} = run_internal_transactions([internal_transaction_changes]) @@ -25,18 +26,21 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactionsTest do transaction = insert(:transaction) |> with_block(status: :ok) pending = insert(:transaction) + insert(:pending_block_operation, block_hash: transaction.block_hash, fetch_internal_transactions: true) + assert :ok == transaction.status assert is_nil(pending.block_hash) index = 0 - transaction_changes = make_internal_transaction_changes(transaction.hash, index, nil) - pending_changes = make_internal_transaction_changes(pending.hash, index, nil) + transaction_changes = make_internal_transaction_changes(transaction, index, nil) + pending_changes = make_internal_transaction_changes(pending, index, nil) assert {:ok, _} = run_internal_transactions([transaction_changes, pending_changes]) - assert %InternalTransaction{} = - Repo.one(from(i in InternalTransaction, where: i.transaction_hash == ^transaction.hash)) + assert Repo.exists?(from(i in InternalTransaction, where: i.transaction_hash == ^transaction.hash)) + + assert PendingBlockOperation |> Repo.get(transaction.block_hash) |> is_nil() assert from(i in InternalTransaction, where: i.transaction_hash == ^pending.hash) |> Repo.one() |> is_nil() @@ -47,68 +51,85 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactionsTest do empty_block = insert(:block) pending = insert(:transaction) + insert(:pending_block_operation, block_hash: empty_block.hash, fetch_internal_transactions: true) + assert is_nil(pending.block_hash) full_block = insert(:block) inserted = insert(:transaction) |> with_block(full_block) + insert(:pending_block_operation, block_hash: full_block.hash, fetch_internal_transactions: true) + assert full_block.hash == inserted.block_hash index = 0 pending_transaction_changes = - pending.hash + pending |> make_internal_transaction_changes(index, nil) |> Map.put(:block_number, empty_block.number) - transaction_changes = - inserted.hash - |> make_internal_transaction_changes(index, nil) - |> Map.put(:block_number, full_block.number) - - multi = - Multi.new() - |> Multi.run(:internal_transactions_indexed_at_blocks, fn _, _ -> {:ok, [empty_block.hash, full_block.hash]} end) + transaction_changes = make_internal_transaction_changes(inserted, index, nil) - assert {:ok, _} = run_internal_transactions([pending_transaction_changes, transaction_changes], multi) + assert {:ok, _} = run_internal_transactions([pending_transaction_changes, transaction_changes]) assert from(i in InternalTransaction, where: i.transaction_hash == ^pending.hash) |> Repo.one() |> is_nil() assert %{consensus: false} = Repo.get(Block, empty_block.hash) + assert not is_nil(Repo.get(PendingBlockOperation, empty_block.hash)) assert from(i in InternalTransaction, where: i.transaction_hash == ^inserted.hash) |> Repo.one() |> is_nil() == false assert %{consensus: true} = Repo.get(Block, full_block.hash) + assert PendingBlockOperation |> Repo.get(full_block.hash) |> is_nil() + end + + test "removes consensus to blocks where not all transactions are filled" do + full_block = insert(:block) + transaction_a = insert(:transaction) |> with_block(full_block) + transaction_b = insert(:transaction) |> with_block(full_block) + + insert(:pending_block_operation, block_hash: full_block.hash, fetch_internal_transactions: true) + + transaction_a_changes = make_internal_transaction_changes(transaction_a, 0, nil) + + assert {:ok, _} = run_internal_transactions([transaction_a_changes]) + + assert from(i in InternalTransaction, where: i.transaction_hash == ^transaction_a.hash) |> Repo.one() |> is_nil() + assert from(i in InternalTransaction, where: i.transaction_hash == ^transaction_b.hash) |> Repo.one() |> is_nil() + + assert %{consensus: false} = Repo.get(Block, full_block.hash) + assert not is_nil(Repo.get(PendingBlockOperation, full_block.hash)) end test "does not remove consensus when block is empty and no transactions are missing" do empty_block = insert(:block) + insert(:pending_block_operation, block_hash: empty_block.hash, fetch_internal_transactions: true) + full_block = insert(:block) inserted = insert(:transaction) |> with_block(full_block) + insert(:pending_block_operation, block_hash: full_block.hash, fetch_internal_transactions: true) + assert full_block.hash == inserted.block_hash index = 0 - transaction_changes = - inserted.hash - |> make_internal_transaction_changes(index, nil) - |> Map.put(:block_number, full_block.number) - - multi = - Multi.new() - |> Multi.run(:internal_transactions_indexed_at_blocks, fn _, _ -> {:ok, [empty_block.hash, full_block.hash]} end) + transaction_changes = make_internal_transaction_changes(inserted, index, nil) + empty_changes = make_empty_block_changes(empty_block.number) - assert {:ok, _} = run_internal_transactions([transaction_changes], multi) + assert {:ok, _} = run_internal_transactions([empty_changes, transaction_changes]) assert %{consensus: true} = Repo.get(Block, empty_block.hash) + assert PendingBlockOperation |> Repo.get(empty_block.hash) |> is_nil() assert from(i in InternalTransaction, where: i.transaction_hash == ^inserted.hash) |> Repo.one() |> is_nil() == false assert %{consensus: true} = Repo.get(Block, full_block.hash) + assert PendingBlockOperation |> Repo.get(full_block.hash) |> is_nil() end end @@ -121,7 +142,9 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactionsTest do |> Repo.transaction() end - defp make_internal_transaction_changes(transaction_hash, index, error) do + defp make_empty_block_changes(block_number), do: %{block_number: block_number} + + defp make_internal_transaction_changes(transaction, index, error) do %{ from_address_hash: insert(:address).hash, to_address_hash: insert(:address).hash, @@ -142,10 +165,11 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactionsTest do end, index: index, trace_address: [], - transaction_hash: transaction_hash, + transaction_hash: transaction.hash, type: :call, value: Wei.from(Decimal.new(1), :wei), - error: error + error: error, + block_number: transaction.block_number } end end diff --git a/apps/explorer/test/explorer/chain/import_test.exs b/apps/explorer/test/explorer/chain/import_test.exs index 4a2fe41c65..468801b77c 100644 --- a/apps/explorer/test/explorer/chain/import_test.exs +++ b/apps/explorer/test/explorer/chain/import_test.exs @@ -12,6 +12,7 @@ defmodule Explorer.Chain.ImportTest do Log, Hash, Import, + PendingBlockOperation, Token, TokenTransfer, Transaction @@ -81,7 +82,8 @@ defmodule Explorer.Chain.ImportTest do value: 0 } ], - timeout: 5 + timeout: 5, + with: :blockless_changeset }, logs: %{ params: [ @@ -306,9 +308,7 @@ defmodule Explorer.Chain.ImportTest do bytes: <<83, 189, 136, 72, 114, 222, 62, 72, 134, 146, 136, 27, 174, 236, 38, 46, 123, 149, 35, 77, 57, 101, 36, 140, 57, 254, 153, 47, 255, 212, 51, 229>> - }, - # because there are successful, non-contract-creation token transfer - internal_transactions_indexed_at: %DateTime{} + } } ], tokens: [ @@ -555,7 +555,8 @@ defmodule Explorer.Chain.ImportTest do block_number: 37, transaction_index: 0 } - ] + ], + with: :blockless_changeset } } @@ -566,63 +567,7 @@ defmodule Explorer.Chain.ImportTest do assert address.contract_code != smart_contract_bytecode end - test "updates `error`, `status` and `internal_transaction_indexed_at` even if internal transactions were alreader inserted" do - address_hash = "0x1c494fa496f1cfd918b5ff190835af3aaf609899" - from_address = insert(:address, hash: address_hash) - - block = insert(:block, consensus: true, number: 37) - - transaction = - :transaction - |> insert(error: nil, internal_transactions_indexed_at: nil, status: nil, from_address: from_address) - |> with_block(block, status: :error) - - internal_transacton = - insert(:internal_transaction, - block_number: 37, - transaction_hash: transaction.hash, - error: "Bad Instruction", - index: 0, - gas_used: nil, - output: nil, - gas: 19, - type: "call" - ) - - options = %{ - internal_transactions: %{ - params: [ - %{ - block_number: internal_transacton.block_number, - call_type: internal_transacton.type, - gas: internal_transacton.gas, - gas_used: internal_transacton.gas_used, - index: internal_transacton.index, - output: internal_transacton.output, - transaction_hash: internal_transacton.transaction_hash, - type: internal_transacton.type, - from_address_hash: address_hash, - to_address_hash: address_hash, - trace_address: [], - value: 0, - transaction_index: 0, - error: internal_transacton.error, - input: internal_transacton.input - } - ] - } - } - - {:ok, _} = Import.all(options) - - assert result = - %Transaction{error: "Bad Instruction", status: :error} = - Repo.one!(from(t in Transaction, where: t.hash == ^transaction.hash)) - - assert result.internal_transactions_indexed_at - end - - test "with internal_transactions updates Transaction internal_transactions_indexed_at" do + test "with internal_transactions updates PendingBlockOperation status" do block_hash = "0xe52d77084cab13a4e724162bcd8c6028e5ecfaa04d091ee476e96b9958ed6b47" block_number = 34 miner_hash = from_address_hash = "0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca" @@ -675,7 +620,10 @@ defmodule Explorer.Chain.ImportTest do value: 0 } ] - }, + } + } + + internal_txs_options = %{ internal_transactions: %{ params: [ %{ @@ -694,17 +642,18 @@ defmodule Explorer.Chain.ImportTest do output: "0x", value: 0 } - ] + ], + with: :blockless_changeset } } - refute Enum.any?(options[:transactions][:params], &Map.has_key?(&1, :internal_transactions_indexed_at)) - assert {:ok, _} = Import.all(options) - transaction = Explorer.Repo.get(Transaction, transaction_hash) + assert [block_hash] = Explorer.Repo.all(PendingBlockOperation.block_hashes(:fetch_internal_transactions)) + + assert {:ok, _} = Import.all(internal_txs_options) - refute transaction.internal_transactions_indexed_at == nil + assert [] == Explorer.Repo.all(PendingBlockOperation.block_hashes(:fetch_internal_transactions)) end test "when the transaction has no to_address and an internal transaction with type create it populates the denormalized created_contract_address_hash" do @@ -787,7 +736,8 @@ defmodule Explorer.Chain.ImportTest do value: 0, transaction_index: 0 } - ] + ], + with: :blockless_changeset } } @@ -897,7 +847,8 @@ defmodule Explorer.Chain.ImportTest do transaction_index: 1 } ], - timeout: 5 + timeout: 5, + with: :blockless_changeset }, addresses: %{ params: [ @@ -1081,7 +1032,8 @@ defmodule Explorer.Chain.ImportTest do transaction_index: 0, transaction_block_number: 35 } - ] + ], + with: :blockless_changeset } }) @@ -1562,7 +1514,8 @@ defmodule Explorer.Chain.ImportTest do transaction_index: 0 ) ], - timeout: 1 + timeout: 1, + with: :blockless_changeset }, logs: %{ params: [params_for(:log, transaction_hash: transaction_hash, address_hash: miner_hash)], @@ -1794,7 +1747,6 @@ defmodule Explorer.Chain.ImportTest do block_hash: block_hash_before, block_number: block_number, error: error, - internal_transactions_indexed_at: Timex.parse!("2019-01-01T01:00:00Z", "{ISO:Extended:Z}"), from_address_hash: from_address_hash_before, to_address_hash: to_address_hash_before, gas: 21_000, @@ -1830,7 +1782,8 @@ defmodule Explorer.Chain.ImportTest do block_number: block_number, transaction_index: 0 } - ] + ], + with: :blockless_changeset } }) diff --git a/apps/explorer/test/explorer/chain/internal_transaction_test.exs b/apps/explorer/test/explorer/chain/internal_transaction_test.exs index fc3977a998..54a1b9f5b1 100644 --- a/apps/explorer/test/explorer/chain/internal_transaction_test.exs +++ b/apps/explorer/test/explorer/chain/internal_transaction_test.exs @@ -26,7 +26,9 @@ defmodule Explorer.Chain.InternalTransactionTest do transaction_hash: transaction.hash, type: "call", value: 100, - block_number: 35 + block_number: 35, + block_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd", + block_index: 0 }) assert changeset.valid? diff --git a/apps/explorer/test/explorer/chain_test.exs b/apps/explorer/test/explorer/chain_test.exs index 148897925f..e67a6ec196 100644 --- a/apps/explorer/test/explorer/chain_test.exs +++ b/apps/explorer/test/explorer/chain_test.exs @@ -393,6 +393,8 @@ defmodule Explorer.ChainTest do transaction: transaction, index: 0, block_number: transaction.block_number, + block_hash: transaction.block_hash, + block_index: 0, transaction_index: transaction.index ) @@ -806,7 +808,7 @@ defmodule Explorer.ChainTest do :transaction |> insert() - |> with_block(block, internal_transactions_indexed_at: DateTime.utc_now()) + |> with_block(block) assert Chain.finished_indexing?() end @@ -822,6 +824,8 @@ defmodule Explorer.ChainTest do |> insert() |> with_block(block) + insert(:pending_block_operation, block: block, fetch_internal_transactions: true) + refute Chain.finished_indexing?() end end @@ -919,6 +923,8 @@ defmodule Explorer.ChainTest do transaction: transaction, index: 0, block_number: transaction.block_number, + block_hash: transaction.block_hash, + block_index: 0, transaction_index: transaction.index ) @@ -927,6 +933,8 @@ defmodule Explorer.ChainTest do transaction: transaction, index: index, block_number: transaction.block_number, + block_hash: transaction.block_hash, + block_index: index, transaction_index: transaction.index ) end) @@ -1281,7 +1289,8 @@ defmodule Explorer.ChainTest do output: "0x", value: 0 } - ] + ], + with: :blockless_changeset }, logs: %{ params: [ @@ -1493,9 +1502,7 @@ defmodule Explorer.ChainTest do bytes: <<83, 189, 136, 72, 114, 222, 62, 72, 134, 146, 136, 27, 174, 236, 38, 46, 123, 149, 35, 77, 57, 101, 36, 140, 57, 254, 153, 47, 255, 212, 51, 229>> - }, - # because there are successful, non-contract-creation token transfer - internal_transactions_indexed_at: %DateTime{} + } } ], tokens: [ @@ -1769,6 +1776,8 @@ defmodule Explorer.ChainTest do transaction: transaction, to_address: address, block_number: transaction.block_number, + block_hash: transaction.block_hash, + block_index: 1, transaction_index: transaction.index ) @@ -1778,6 +1787,8 @@ defmodule Explorer.ChainTest do transaction: transaction, to_address: address, block_number: transaction.block_number, + block_hash: transaction.block_hash, + block_index: 2, transaction_index: transaction.index ) @@ -1804,6 +1815,8 @@ defmodule Explorer.ChainTest do to_address: address, index: 0, block_number: transaction.block_number, + block_hash: transaction.block_hash, + block_index: 0, transaction_index: transaction.index ) @@ -1812,6 +1825,8 @@ defmodule Explorer.ChainTest do to_address: address, index: 1, block_number: transaction.block_number, + block_hash: transaction.block_hash, + block_index: 1, transaction_index: transaction.index ) @@ -1859,6 +1874,8 @@ defmodule Explorer.ChainTest do to_address: address, index: 1, block_number: pending_transaction.block_number, + block_hash: pending_transaction.block_hash, + block_index: 1, transaction_index: pending_transaction.index ) @@ -1869,6 +1886,8 @@ defmodule Explorer.ChainTest do to_address: address, index: 2, block_number: pending_transaction.block_number, + block_hash: pending_transaction.block_hash, + block_index: 2, transaction_index: pending_transaction.index ) @@ -1886,6 +1905,8 @@ defmodule Explorer.ChainTest do to_address: address, index: 1, block_number: first_a_transaction.block_number, + block_hash: a_block.hash, + block_index: 1, transaction_index: first_a_transaction.index ) @@ -1896,6 +1917,8 @@ defmodule Explorer.ChainTest do to_address: address, index: 2, block_number: first_a_transaction.block_number, + block_hash: a_block.hash, + block_index: 2, transaction_index: first_a_transaction.index ) @@ -1911,6 +1934,8 @@ defmodule Explorer.ChainTest do to_address: address, index: 1, block_number: second_a_transaction.block_number, + block_hash: a_block.hash, + block_index: 4, transaction_index: second_a_transaction.index ) @@ -1921,6 +1946,8 @@ defmodule Explorer.ChainTest do to_address: address, index: 2, block_number: second_a_transaction.block_number, + block_hash: a_block.hash, + block_index: 5, transaction_index: second_a_transaction.index ) @@ -1938,6 +1965,8 @@ defmodule Explorer.ChainTest do to_address: address, index: 1, block_number: first_b_transaction.block_number, + block_hash: b_block.hash, + block_index: 1, transaction_index: first_b_transaction.index ) @@ -1948,6 +1977,8 @@ defmodule Explorer.ChainTest do to_address: address, index: 2, block_number: first_b_transaction.block_number, + block_hash: b_block.hash, + block_index: 2, transaction_index: first_b_transaction.index ) @@ -1973,10 +2004,14 @@ defmodule Explorer.ChainTest do pending_transaction = insert(:transaction) + old_block = insert(:block, consensus: false) + insert( :internal_transaction, transaction: pending_transaction, to_address: address, + block_hash: old_block.hash, + block_index: 1, index: 1 ) @@ -1984,6 +2019,8 @@ defmodule Explorer.ChainTest do :internal_transaction, transaction: pending_transaction, to_address: address, + block_hash: old_block.hash, + block_index: 2, index: 2 ) @@ -2001,6 +2038,8 @@ defmodule Explorer.ChainTest do to_address: address, index: 1, block_number: first_a_transaction.block_number, + block_hash: a_block.hash, + block_index: 1, transaction_index: first_a_transaction.index ) @@ -2011,6 +2050,8 @@ defmodule Explorer.ChainTest do to_address: address, index: 2, block_number: first_a_transaction.block_number, + block_hash: a_block.hash, + block_index: 2, transaction_index: first_a_transaction.index ) @@ -2026,6 +2067,8 @@ defmodule Explorer.ChainTest do to_address: address, index: 1, block_number: second_a_transaction.block_number, + block_hash: a_block.hash, + block_index: 4, transaction_index: second_a_transaction.index ) @@ -2036,6 +2079,8 @@ defmodule Explorer.ChainTest do to_address: address, index: 2, block_number: second_a_transaction.block_number, + block_hash: a_block.hash, + block_index: 5, transaction_index: second_a_transaction.index ) @@ -2053,6 +2098,8 @@ defmodule Explorer.ChainTest do to_address: address, index: 1, block_number: first_b_transaction.block_number, + block_hash: b_block.hash, + block_index: 1, transaction_index: first_b_transaction.index ) @@ -2063,6 +2110,8 @@ defmodule Explorer.ChainTest do to_address: address, index: 2, block_number: first_b_transaction.block_number, + block_hash: b_block.hash, + block_index: 2, transaction_index: first_b_transaction.index ) @@ -2130,6 +2179,8 @@ defmodule Explorer.ChainTest do to_address: address, transaction: transaction, block_number: transaction.block_number, + block_hash: transaction.block_hash, + block_index: 0, transaction_index: transaction.index ) @@ -2150,6 +2201,8 @@ defmodule Explorer.ChainTest do index: 0, from_address: address, transaction: transaction, + block_hash: transaction.block_hash, + block_index: 0, block_number: transaction.block_number, transaction_index: transaction.index ) @@ -2213,6 +2266,8 @@ defmodule Explorer.ChainTest do transaction: transaction, index: 0, block_number: transaction.block_number, + block_hash: transaction.block_hash, + block_index: 0, transaction_index: transaction.index ) @@ -2220,6 +2275,8 @@ defmodule Explorer.ChainTest do insert(:internal_transaction, transaction: transaction, index: 1, + block_hash: transaction.block_hash, + block_index: 1, block_number: transaction.block_number, transaction_index: transaction.index ) @@ -2249,6 +2306,8 @@ defmodule Explorer.ChainTest do transaction: transaction, index: 0, block_number: transaction.block_number, + block_hash: transaction.block_hash, + block_index: 0, transaction_index: transaction.index ) @@ -2287,6 +2346,8 @@ defmodule Explorer.ChainTest do transaction: transaction, index: 0, block_number: transaction.block_number, + block_hash: transaction.block_hash, + block_index: 0, transaction_index: transaction.index ) @@ -2306,6 +2367,8 @@ defmodule Explorer.ChainTest do index: 0, transaction: transaction, block_number: transaction.block_number, + block_hash: transaction.block_hash, + block_index: 0, transaction_index: transaction.index ) @@ -2326,6 +2389,8 @@ defmodule Explorer.ChainTest do transaction: transaction, type: :reward, block_number: transaction.block_number, + block_hash: transaction.block_hash, + block_index: 0, transaction_index: transaction.index ) @@ -2347,6 +2412,8 @@ defmodule Explorer.ChainTest do gas: nil, type: :selfdestruct, block_number: transaction.block_number, + block_hash: transaction.block_hash, + block_index: 0, transaction_index: transaction.index ) @@ -2366,6 +2433,8 @@ defmodule Explorer.ChainTest do transaction: transaction, index: 0, block_number: transaction.block_number, + block_hash: transaction.block_hash, + block_index: 0, transaction_index: transaction.index ) @@ -2374,6 +2443,8 @@ defmodule Explorer.ChainTest do transaction: transaction, index: 1, block_number: transaction.block_number, + block_hash: transaction.block_hash, + block_index: 1, transaction_index: transaction.index ) @@ -2396,6 +2467,8 @@ defmodule Explorer.ChainTest do transaction: transaction, index: 0, block_number: transaction.block_number, + block_hash: transaction.block_hash, + block_index: 0, transaction_index: transaction.index ) @@ -2404,6 +2477,8 @@ defmodule Explorer.ChainTest do transaction: transaction, index: 1, block_number: transaction.block_number, + block_hash: transaction.block_hash, + block_index: 1, transaction_index: transaction.index ) @@ -2878,6 +2953,8 @@ defmodule Explorer.ChainTest do created_contract_address: created_contract_address, created_contract_code: smart_contract_bytecode, block_number: transaction.block_number, + block_hash: transaction.block_hash, + block_index: 0, transaction_index: transaction.index ) @@ -2978,6 +3055,8 @@ defmodule Explorer.ChainTest do created_contract_address: created_contract_address, created_contract_code: smart_contract_bytecode, block_number: transaction.block_number, + block_hash: transaction.block_hash, + block_index: 0, transaction_index: transaction.index ) @@ -3184,6 +3263,8 @@ defmodule Explorer.ChainTest do index: 0, transaction: transaction, block_number: transaction.block_number, + block_hash: transaction.block_hash, + block_index: 0, transaction_index: transaction.index ) @@ -3230,6 +3311,8 @@ defmodule Explorer.ChainTest do index: 0, transaction: transaction, block_number: transaction.block_number, + block_hash: transaction.block_hash, + block_index: 0, transaction_index: transaction.index ) @@ -3270,6 +3353,8 @@ defmodule Explorer.ChainTest do index: 0, transaction: transaction, block_number: transaction.block_number, + block_hash: transaction.block_hash, + block_index: 0, transaction_index: transaction.index ) @@ -3339,6 +3424,8 @@ defmodule Explorer.ChainTest do index: 0, transaction: from_internal_transaction_transaction, block_number: from_internal_transaction_transaction.block_number, + block_hash: from_internal_transaction_transaction.block_hash, + block_index: 0, transaction_index: from_internal_transaction_transaction.index ) @@ -3357,6 +3444,8 @@ defmodule Explorer.ChainTest do to_address: miner, transaction: to_internal_transaction_transaction, block_number: to_internal_transaction_transaction.block_number, + block_hash: to_internal_transaction_transaction.block_hash, + block_index: 0, transaction_index: to_internal_transaction_transaction.index ) @@ -3413,6 +3502,8 @@ defmodule Explorer.ChainTest do index: 0, transaction: from_internal_transaction_transaction, block_number: from_internal_transaction_transaction.block_number, + block_hash: from_internal_transaction_transaction.block_hash, + block_index: 0, transaction_index: from_internal_transaction_transaction.index ) @@ -3427,6 +3518,8 @@ defmodule Explorer.ChainTest do index: 0, transaction: to_internal_transaction_transaction, block_number: to_internal_transaction_transaction.block_number, + block_hash: to_internal_transaction_transaction.block_hash, + block_index: 1, transaction_index: to_internal_transaction_transaction.index ) @@ -4166,6 +4259,8 @@ defmodule Explorer.ChainTest do index: 0, created_contract_address: created_contract_address, block_number: transaction.block_number, + block_hash: transaction.block_hash, + block_index: 0, transaction_index: transaction.index, input: input ) diff --git a/apps/explorer/test/explorer/etherscan_test.exs b/apps/explorer/test/explorer/etherscan_test.exs index 7c1e07a438..08ac3eaab1 100644 --- a/apps/explorer/test/explorer/etherscan_test.exs +++ b/apps/explorer/test/explorer/etherscan_test.exs @@ -68,6 +68,8 @@ defmodule Explorer.EtherscanTest do transaction: transaction, index: 0, block_number: transaction.block_number, + block_hash: transaction.block_hash, + block_index: 0, transaction_index: transaction.index ) |> with_contract_creation(contract_address) @@ -144,6 +146,8 @@ defmodule Explorer.EtherscanTest do transaction: transaction, index: 0, block_number: transaction.block_number, + block_hash: transaction.block_hash, + block_index: 0, transaction_index: transaction.index ) |> with_contract_creation(contract_address) @@ -485,6 +489,8 @@ defmodule Explorer.EtherscanTest do index: 0, from_address: address, block_number: transaction.block_number, + block_hash: transaction.block_hash, + block_index: 0, transaction_index: transaction.index ) |> with_contract_creation(contract_address) @@ -527,6 +533,8 @@ defmodule Explorer.EtherscanTest do transaction: transaction, index: index, block_number: transaction.block_number, + block_hash: transaction.block_hash, + block_index: index, transaction_index: transaction.index ) end @@ -551,6 +559,8 @@ defmodule Explorer.EtherscanTest do transaction: transaction1, index: 0, block_number: transaction1.block_number, + block_hash: transaction1.block_hash, + block_index: 0, transaction_index: transaction1.index ) @@ -558,6 +568,8 @@ defmodule Explorer.EtherscanTest do transaction: transaction1, index: 1, block_number: transaction1.block_number, + block_hash: transaction1.block_hash, + block_index: 1, transaction_index: transaction1.index ) @@ -566,6 +578,8 @@ defmodule Explorer.EtherscanTest do index: 0, type: :reward, block_number: transaction2.block_number, + block_hash: transaction2.block_hash, + block_index: 2, transaction_index: transaction2.index ) @@ -617,6 +631,8 @@ defmodule Explorer.EtherscanTest do index: 0, from_address: address, block_number: transaction.block_number, + block_hash: block.hash, + block_index: 0, transaction_index: transaction.index ) |> with_contract_creation(contract_address) @@ -665,6 +681,8 @@ defmodule Explorer.EtherscanTest do index: index, from_address: address, block_number: transaction.block_number, + block_hash: transaction.block_hash, + block_index: index, transaction_index: transaction.index } @@ -689,6 +707,8 @@ defmodule Explorer.EtherscanTest do transaction: transaction, index: 0, block_number: transaction.block_number, + block_hash: transaction.block_hash, + block_index: 0, transaction_index: transaction.index, created_contract_address: address1 ) @@ -697,6 +717,8 @@ defmodule Explorer.EtherscanTest do transaction: transaction, index: 1, block_number: transaction.block_number, + block_hash: transaction.block_hash, + block_index: 1, transaction_index: transaction.index, from_address: address1 ) @@ -705,6 +727,8 @@ defmodule Explorer.EtherscanTest do transaction: transaction, index: 2, block_number: transaction.block_number, + block_hash: transaction.block_hash, + block_index: 2, transaction_index: transaction.index, to_address: address1 ) @@ -713,6 +737,8 @@ defmodule Explorer.EtherscanTest do transaction: transaction, index: 3, block_number: transaction.block_number, + block_hash: transaction.block_hash, + block_index: 3, transaction_index: transaction.index, from_address: address2 ) @@ -740,6 +766,8 @@ defmodule Explorer.EtherscanTest do index: index, from_address: address, block_number: transaction.block_number, + block_hash: transaction.block_hash, + block_index: index, transaction_index: transaction.index } @@ -780,6 +808,8 @@ defmodule Explorer.EtherscanTest do index: index, from_address: address, block_number: transaction.block_number, + block_hash: transaction.block_hash, + block_index: index, transaction_index: transaction.index } diff --git a/apps/explorer/test/explorer/graphql_test.exs b/apps/explorer/test/explorer/graphql_test.exs index 41dd008dae..357f7fe94c 100644 --- a/apps/explorer/test/explorer/graphql_test.exs +++ b/apps/explorer/test/explorer/graphql_test.exs @@ -92,9 +92,15 @@ defmodule Explorer.GraphQLTest do describe "get_internal_transaction/1" do test "returns existing internal transaction" do - transaction = insert(:transaction) + transaction = insert(:transaction) |> with_block() - internal_transaction = insert(:internal_transaction, transaction: transaction, index: 0) + internal_transaction = + insert(:internal_transaction, + transaction: transaction, + index: 0, + block_hash: transaction.block_hash, + block_index: 0 + ) clauses = %{transaction_hash: transaction.hash, index: internal_transaction.index} @@ -117,11 +123,23 @@ defmodule Explorer.GraphQLTest do describe "transcation_to_internal_transactions_query/1" do test "with transaction with one internal transaction" do - transaction1 = insert(:transaction) - transaction2 = insert(:transaction) - - internal_transaction = insert(:internal_transaction_create, transaction: transaction1, index: 0) - insert(:internal_transaction_create, transaction: transaction2, index: 0) + transaction1 = insert(:transaction) |> with_block() + transaction2 = insert(:transaction) |> with_block() + + internal_transaction = + insert(:internal_transaction_create, + transaction: transaction1, + index: 0, + block_hash: transaction1.block_hash, + block_index: 0 + ) + + insert(:internal_transaction_create, + transaction: transaction2, + index: 0, + block_hash: transaction2.block_hash, + block_index: 0 + ) [found_internal_transaction] = transaction1 @@ -133,14 +151,24 @@ defmodule Explorer.GraphQLTest do end test "with transaction with multiple internal transactions" do - transaction1 = insert(:transaction) - transaction2 = insert(:transaction) + transaction1 = insert(:transaction) |> with_block() + transaction2 = insert(:transaction) |> with_block() for index <- 0..2 do - insert(:internal_transaction_create, transaction: transaction1, index: index) + insert(:internal_transaction_create, + transaction: transaction1, + index: index, + block_hash: transaction1.block_hash, + block_index: index + ) end - insert(:internal_transaction_create, transaction: transaction2, index: 0) + insert(:internal_transaction_create, + transaction: transaction2, + index: 0, + block_hash: transaction2.block_hash, + block_index: 0 + ) found_internal_transactions = transaction1 @@ -155,11 +183,28 @@ defmodule Explorer.GraphQLTest do end test "orders internal transactions by ascending index" do - transaction = insert(:transaction) - - insert(:internal_transaction_create, transaction: transaction, index: 2) - insert(:internal_transaction_create, transaction: transaction, index: 0) - insert(:internal_transaction_create, transaction: transaction, index: 1) + transaction = insert(:transaction) |> with_block() + + insert(:internal_transaction_create, + transaction: transaction, + index: 2, + block_hash: transaction.block_hash, + block_index: 2 + ) + + insert(:internal_transaction_create, + transaction: transaction, + index: 0, + block_hash: transaction.block_hash, + block_index: 0 + ) + + insert(:internal_transaction_create, + transaction: transaction, + index: 1, + block_hash: transaction.block_hash, + block_index: 1 + ) found_internal_transactions = transaction diff --git a/apps/explorer/test/explorer/repo_test.exs b/apps/explorer/test/explorer/repo_test.exs index e761bdfd23..e414f8745d 100644 --- a/apps/explorer/test/explorer/repo_test.exs +++ b/apps/explorer/test/explorer/repo_test.exs @@ -10,7 +10,7 @@ defmodule Explorer.RepoTest do describe "safe_insert_all/3" do test "inserting duplicate rows in one chunk is logged before re-raising exception" do - transaction = insert(:transaction) + transaction = insert(:transaction) |> with_block() params = params_for( @@ -20,6 +20,8 @@ defmodule Explorer.RepoTest do transaction_hash: transaction.hash, index: 0, block_number: 35, + block_hash: transaction.block_hash, + block_index: 0, transaction_index: 0 ) @@ -33,7 +35,7 @@ defmodule Explorer.RepoTest do Repo.safe_insert_all( InternalTransaction, [timestamped_changes, timestamped_changes], - conflict_target: [:transaction_hash, :index], + conflict_target: [:block_hash, :block_index], on_conflict: :replace_all ) end @@ -42,7 +44,7 @@ defmodule Explorer.RepoTest do assert log =~ "Chunk:\n" assert log =~ "index: 0" - assert log =~ "Options:\n\n[conflict_target: [:transaction_hash, :index], on_conflict: :replace_all]\n\n" + assert log =~ "Options:\n\n[conflict_target: [:block_hash, :block_index], on_conflict: :replace_all]\n\n" assert log =~ "Exception:\n\n** (Postgrex.Error) ERROR 21000 (cardinality_violation) ON CONFLICT DO UPDATE command cannot affect row a second time\n" diff --git a/apps/explorer/test/support/factory.ex b/apps/explorer/test/support/factory.ex index 1d560630df..b26815c1f9 100644 --- a/apps/explorer/test/support/factory.ex +++ b/apps/explorer/test/support/factory.ex @@ -23,6 +23,7 @@ defmodule Explorer.Factory do Hash, InternalTransaction, Log, + PendingBlockOperation, SmartContract, Token, TokenTransfer, @@ -226,15 +227,9 @@ defmodule Explorer.Factory do cumulative_gas_used = collated_params[:cumulative_gas_used] || Enum.random(21_000..100_000) gas_used = collated_params[:gas_used] || Enum.random(21_000..100_000) - internal_transactions_indexed_at = collated_params[:internal_transactions_indexed_at] status = Keyword.get(collated_params, :status, Enum.random([:ok, :error])) - error = - if internal_transactions_indexed_at != nil && status == :error do - collated_params[:error] || "Something really bad happened" - else - nil - end + error = (status == :error && collated_params[:error]) || nil transaction |> Transaction.changeset(%{ @@ -244,7 +239,6 @@ defmodule Explorer.Factory do error: error, gas_used: gas_used, index: next_transaction_index, - internal_transactions_indexed_at: internal_transactions_indexed_at, status: status }) |> Repo.update!() @@ -290,6 +284,14 @@ defmodule Explorer.Factory do data end + def pending_block_operation_factory do + %PendingBlockOperation{ + # caller MUST supply block + # all operations will default to false + fetch_internal_transactions: false + } + end + def internal_transaction_factory() do gas = Enum.random(21_000..100_000) gas_used = Enum.random(0..gas) @@ -306,6 +308,8 @@ defmodule Explorer.Factory do trace_address: [], # caller MUST supply `transaction` because it can't be built lazily to allow overrides without creating an extra # transaction + # caller MUST supply `block_hash` (usually the same as the transaction's) + # caller MUST supply `block_index` type: :call, value: sequence("internal_transaction_value", &Decimal.new(&1)) } @@ -328,6 +332,8 @@ defmodule Explorer.Factory do trace_address: [], # caller MUST supply `transaction` because it can't be built lazily to allow overrides without creating an extra # transaction + # caller MUST supply `block_hash` (usually the same as the transaction's) + # caller MUST supply `block_index` type: :create, value: sequence("internal_transaction_value", &Decimal.new(&1)) } diff --git a/apps/indexer/lib/indexer/block/catchup/fetcher.ex b/apps/indexer/lib/indexer/block/catchup/fetcher.ex index d158443be1..d15d887351 100644 --- a/apps/indexer/lib/indexer/block/catchup/fetcher.ex +++ b/apps/indexer/lib/indexer/block/catchup/fetcher.ex @@ -12,7 +12,7 @@ defmodule Indexer.Block.Catchup.Fetcher do async_import_block_rewards: 1, async_import_coin_balances: 2, async_import_created_contract_codes: 1, - async_import_internal_transactions: 2, + async_import_internal_transactions: 1, async_import_replaced_transactions: 1, async_import_tokens: 1, async_import_token_balances: 1, @@ -122,7 +122,7 @@ defmodule Indexer.Block.Catchup.Fetcher do @async_import_remaining_block_data_options ~w(address_hash_to_fetched_balance_block_number)a @impl Block.Fetcher - def import(%Block.Fetcher{json_rpc_named_arguments: json_rpc_named_arguments}, options) when is_map(options) do + def import(_block_fetcher, options) when is_map(options) do {async_import_remaining_block_data_options, options_with_block_rewards_errors} = Map.split(options, @async_import_remaining_block_data_options) @@ -135,8 +135,7 @@ defmodule Indexer.Block.Catchup.Fetcher do with {:import, {:ok, imported} = ok} <- {:import, Chain.import(full_chain_import_options)} do async_import_remaining_block_data( imported, - Map.put(async_import_remaining_block_data_options, :block_rewards, %{errors: block_reward_errors}), - json_rpc_named_arguments + Map.put(async_import_remaining_block_data_options, :block_rewards, %{errors: block_reward_errors}) ) ok @@ -145,13 +144,12 @@ defmodule Indexer.Block.Catchup.Fetcher do defp async_import_remaining_block_data( imported, - %{block_rewards: %{errors: block_reward_errors}} = options, - json_rpc_named_arguments + %{block_rewards: %{errors: block_reward_errors}} = options ) do async_import_block_rewards(block_reward_errors) async_import_coin_balances(imported, options) async_import_created_contract_codes(imported) - async_import_internal_transactions(imported, Keyword.get(json_rpc_named_arguments, :variant)) + async_import_internal_transactions(imported) async_import_tokens(imported) async_import_token_balances(imported) async_import_uncles(imported) diff --git a/apps/indexer/lib/indexer/block/fetcher.ex b/apps/indexer/lib/indexer/block/fetcher.ex index 9a2127ae15..195c289f6c 100644 --- a/apps/indexer/lib/indexer/block/fetcher.ex +++ b/apps/indexer/lib/indexer/block/fetcher.ex @@ -10,7 +10,6 @@ defmodule Indexer.Block.Fetcher do import EthereumJSONRPC, only: [quantity_to_integer: 1] alias EthereumJSONRPC.{Blocks, FetchedBeneficiaries} - alias Explorer.Chain alias Explorer.Chain.{Address, Block, Hash, Import, Transaction} alias Explorer.Chain.Cache.Blocks, as: BlocksCache alias Explorer.Chain.Cache.{Accounts, BlockNumber, PendingTransactions, Transactions, Uncles} @@ -71,7 +70,6 @@ defmodule Indexer.Block.Fetcher do @receipts_batch_size 250 @receipts_concurrency 10 - @geth_block_limit 128 @doc false def default_receipts_batch_size, do: @receipts_batch_size @@ -264,14 +262,10 @@ defmodule Indexer.Block.Fetcher do block_number: block_number, hash: hash, created_contract_address_hash: %Hash{} = created_contract_address_hash, - created_contract_code_indexed_at: nil, - internal_transactions_indexed_at: nil + created_contract_code_indexed_at: nil } -> [%{block_number: block_number, hash: hash, created_contract_address_hash: created_contract_address_hash}] - %Transaction{internal_transactions_indexed_at: %DateTime{}} -> - [] - %Transaction{created_contract_address_hash: nil} -> [] end) @@ -280,30 +274,13 @@ defmodule Indexer.Block.Fetcher do def async_import_created_contract_codes(_), do: :ok - def async_import_internal_transactions(%{blocks: blocks}, EthereumJSONRPC.Parity) do + def async_import_internal_transactions(%{blocks: blocks}) do blocks - |> Enum.map(fn %Block{number: block_number} -> %{number: block_number} end) - |> InternalTransaction.async_block_fetch(10_000) - end - - def async_import_internal_transactions(%{transactions: transactions}, EthereumJSONRPC.Geth) do - max_block_number = Chain.fetch_max_block_number() - - transactions - |> Enum.flat_map(fn - %Transaction{block_number: block_number, index: index, hash: hash, internal_transactions_indexed_at: nil} -> - [%{block_number: block_number, index: index, hash: hash}] - - %Transaction{internal_transactions_indexed_at: %DateTime{}} -> - [] - end) - |> Enum.filter(fn %{block_number: block_number} -> - max_block_number - block_number < @geth_block_limit - end) + |> Enum.map(fn %Block{number: block_number} -> block_number end) |> InternalTransaction.async_fetch(10_000) end - def async_import_internal_transactions(_, _), do: :ok + def async_import_internal_transactions(_), do: :ok def async_import_tokens(%{tokens: tokens}) do tokens diff --git a/apps/indexer/lib/indexer/block/realtime/fetcher.ex b/apps/indexer/lib/indexer/block/realtime/fetcher.ex index 5767270f87..572eee7dc7 100644 --- a/apps/indexer/lib/indexer/block/realtime/fetcher.ex +++ b/apps/indexer/lib/indexer/block/realtime/fetcher.ex @@ -15,7 +15,7 @@ defmodule Indexer.Block.Realtime.Fetcher do only: [ async_import_block_rewards: 1, async_import_created_contract_codes: 1, - async_import_internal_transactions: 2, + async_import_internal_transactions: 1, async_import_replaced_transactions: 1, async_import_tokens: 1, async_import_token_balances: 1, @@ -183,7 +183,7 @@ defmodule Indexer.Block.Realtime.Fetcher do @impl Block.Fetcher def import( - %Block.Fetcher{json_rpc_named_arguments: json_rpc_named_arguments} = block_fetcher, + block_fetcher, %{ address_coin_balances: %{params: address_coin_balances_params}, address_hash_to_fetched_balance_block_number: address_hash_to_block_number, @@ -209,8 +209,7 @@ defmodule Indexer.Block.Realtime.Fetcher do {:import, {:ok, imported} = ok} <- {:import, Chain.import(chain_import_options)} do async_import_remaining_block_data( imported, - %{block_rewards: %{errors: block_reward_errors}}, - json_rpc_named_arguments + %{block_rewards: %{errors: block_reward_errors}} ) Accounts.drop(imported[:addresses]) @@ -381,12 +380,11 @@ defmodule Indexer.Block.Realtime.Fetcher do defp async_import_remaining_block_data( imported, - %{block_rewards: %{errors: block_reward_errors}}, - json_rpc_named_arguments + %{block_rewards: %{errors: block_reward_errors}} ) do async_import_block_rewards(block_reward_errors) async_import_created_contract_codes(imported) - async_import_internal_transactions(imported, Keyword.get(json_rpc_named_arguments, :variant)) + async_import_internal_transactions(imported) async_import_tokens(imported) async_import_token_balances(imported) async_import_token_instances(imported) diff --git a/apps/indexer/lib/indexer/fetcher/internal_transaction.ex b/apps/indexer/lib/indexer/fetcher/internal_transaction.ex index 77ba3cd6ec..db93c8d7e4 100644 --- a/apps/indexer/lib/indexer/fetcher/internal_transaction.ex +++ b/apps/indexer/lib/indexer/fetcher/internal_transaction.ex @@ -46,34 +46,9 @@ defmodule Indexer.Fetcher.InternalTransaction do *Note*: The internal transactions for individual transactions cannot be paginated, so the total number of internal transactions that could be produced is unknown. """ - @spec async_fetch([%{required(:block_number) => Block.block_number(), required(:hash) => Hash.Full.t()}]) :: :ok - def async_fetch(transactions_fields, timeout \\ 5000) when is_list(transactions_fields) do - entries = Enum.map(transactions_fields, &entry/1) - - BufferedTask.buffer(__MODULE__, entries, timeout) - end - - @doc """ - Asynchronously fetches internal transactions. - - ## Limiting Upstream Load - - Internal transactions are an expensive upstream operation. The number of - results to fetch is configured by `@max_batch_size` and represents the number - of transaction hashes to request internal transactions in a single JSONRPC - request. Defaults to `#{@max_batch_size}`. - - The `@max_concurrency` attribute configures the number of concurrent requests - of `@max_batch_size` to allow against the JSONRPC. Defaults to `#{@max_concurrency}`. - - *Note*: The internal transactions for individual transactions cannot be paginated, - so the total number of internal transactions that could be produced is unknown. - """ - @spec async_block_fetch([%{required(:block_number) => Block.block_number()}]) :: :ok - def async_block_fetch(transactions_fields, timeout \\ 5000) when is_list(transactions_fields) do - entries = Enum.map(transactions_fields, &block_entry/1) - - BufferedTask.buffer(__MODULE__, entries, timeout) + @spec async_fetch([Block.block_number()]) :: :ok + def async_fetch(block_numbers, timeout \\ 5000) when is_list(block_numbers) do + BufferedTask.buffer(__MODULE__, block_numbers, timeout) end @doc false @@ -95,52 +70,19 @@ defmodule Indexer.Fetcher.InternalTransaction do end @impl BufferedTask - def init(initial, reducer, json_rpc_named_arguments) do + def init(initial, reducer, _json_rpc_named_arguments) do {:ok, final} = - case Keyword.fetch!(json_rpc_named_arguments, :variant) do - EthereumJSONRPC.Parity -> - Chain.stream_blocks_with_unfetched_internal_transactions( - [:number], - initial, - fn block_fields, acc -> - block_fields - |> block_entry() - |> reducer.(acc) - end - ) - - _ -> - Chain.stream_transactions_with_unfetched_internal_transactions( - [:block_number, :hash, :index], - initial, - fn transaction_fields, acc -> - transaction_fields - |> entry() - |> reducer.(acc) - end - ) - end + Chain.stream_blocks_with_unfetched_internal_transactions(initial, fn block_number, acc -> + reducer.(block_number, acc) + end) final end - defp entry(%{block_number: block_number, hash: %Hash{bytes: bytes}, index: index}) when is_integer(block_number) do - {block_number, bytes, index} - end - - defp params({block_number, hash_bytes, index}) when is_integer(block_number) do - {:ok, hash} = Hash.Full.cast(hash_bytes) + defp params(%{block_number: block_number, hash: hash, index: index}) when is_integer(block_number) do %{block_number: block_number, hash_data: to_string(hash), transaction_index: index} end - defp block_entry(%{number: block_number}) when is_integer(block_number) do - block_number - end - - defp block_params(block_number) when is_integer(block_number) do - %{number: block_number} - end - @impl BufferedTask @decorate trace( name: "fetch", @@ -148,52 +90,62 @@ defmodule Indexer.Fetcher.InternalTransaction do service: :indexer, tracer: Tracer ) - def run(entries, json_rpc_named_arguments) do - variant = Keyword.fetch!(json_rpc_named_arguments, :variant) - - unique_entries = unique_entries(entries, variant) + def run(block_numbers, json_rpc_named_arguments) do + unique_numbers = Enum.uniq(block_numbers) - unique_entries_count = Enum.count(unique_entries) - Logger.metadata(count: unique_entries_count) + unique_numbers_count = Enum.count(unique_numbers) + Logger.metadata(count: unique_numbers_count) - Logger.debug("fetching internal transactions for transactions") + Logger.debug("fetching internal transactions for blocks") - variant + json_rpc_named_arguments + |> Keyword.fetch!(:variant) |> case do EthereumJSONRPC.Parity -> - unique_entries - |> EthereumJSONRPC.fetch_block_internal_transactions(json_rpc_named_arguments) + EthereumJSONRPC.fetch_block_internal_transactions(unique_numbers, json_rpc_named_arguments) _ -> - unique_entries - |> Enum.map(¶ms/1) - |> EthereumJSONRPC.fetch_internal_transactions(json_rpc_named_arguments) + fetch_block_internal_transactions_by_transactions(unique_numbers, json_rpc_named_arguments) end |> case do {:ok, internal_transactions_params} -> - import_internal_transaction(internal_transactions_params, json_rpc_named_arguments, unique_entries) + import_internal_transaction(internal_transactions_params, unique_numbers) {:error, reason} -> - Logger.error(fn -> ["failed to fetch internal transactions for transactions: ", inspect(reason)] end, - error_count: unique_entries_count + Logger.error(fn -> ["failed to fetch internal transactions for blocks: ", inspect(reason)] end, + error_count: unique_numbers_count ) # re-queue the de-duped entries - {:retry, unique_entries} + {:retry, unique_numbers} :ignore -> :ok end end - defp import_internal_transaction(internal_transactions_params, json_rpc_named_arguments, unique_entries) do - internal_transactions_indexed_at_blocks = - case Keyword.fetch!(json_rpc_named_arguments, :variant) do - EthereumJSONRPC.Parity -> Enum.map(unique_entries, &block_params/1) - _ -> [] - end + defp fetch_block_internal_transactions_by_transactions(unique_numbers, json_rpc_named_arguments) do + Enum.reduce(unique_numbers, {:ok, []}, fn + block_number, {:ok, acc_list} -> + block_number + |> Chain.get_transactions_of_block_number() + |> Enum.map(¶ms(&1)) + |> case do + [] -> {:ok, []} + transactions -> EthereumJSONRPC.fetch_internal_transactions(transactions, json_rpc_named_arguments) + end + |> case do + {:ok, internal_transactions} -> {:ok, internal_transactions ++ acc_list} + error_or_ignore -> error_or_ignore + end + + _, error_or_ignore -> + error_or_ignore + end) + end - unique_entries_count = Enum.count(unique_entries) + defp import_internal_transaction(internal_transactions_params, unique_numbers) do + unique_numbers_count = Enum.count(unique_numbers) internal_transactions_params_without_failed_creations = remove_failed_creations(internal_transactions_params) addresses_params = @@ -206,14 +158,19 @@ defmodule Indexer.Fetcher.InternalTransaction do {hash, block_number} end) + empty_block_numbers = + unique_numbers + |> MapSet.new() + |> MapSet.difference(MapSet.new(internal_transactions_params_without_failed_creations, & &1.block_number)) + |> Enum.map(&%{block_number: &1}) + + internal_transactions_and_empty_block_numbers = + internal_transactions_params_without_failed_creations ++ empty_block_numbers + imports = Chain.import(%{ addresses: %{params: addresses_params}, - internal_transactions: %{params: internal_transactions_params_without_failed_creations}, - internal_transactions_indexed_at_blocks: %{ - params: internal_transactions_indexed_at_blocks, - with: :number_only_changeset - }, + internal_transactions: %{params: internal_transactions_and_empty_block_numbers, with: :blockless_changeset}, timeout: :infinity }) @@ -230,60 +187,19 @@ defmodule Indexer.Fetcher.InternalTransaction do Logger.error( fn -> [ - "failed to import internal transactions for transactions: ", + "failed to import internal transactions for blocks: ", inspect(reason) ] end, step: step, - error_count: unique_entries_count + error_count: unique_numbers_count ) # re-queue the de-duped entries - {:retry, unique_entries} + {:retry, unique_numbers} end end - defp unique_entries(entries, EthereumJSONRPC.Parity), do: Enum.uniq(entries) - - # Protection and improved reporting for https://github.com/poanetwork/blockscout/issues/289 - defp unique_entries(entries, _) do - entries_by_hash_bytes = Enum.group_by(entries, &elem(&1, 1)) - - if map_size(entries_by_hash_bytes) < length(entries) do - {unique_entries, duplicate_entries} = - entries_by_hash_bytes - |> Map.values() - |> uniques_and_duplicates() - - Logger.error(fn -> - duplicate_entries - |> Stream.with_index() - |> Enum.reduce( - ["Duplicate entries being used to fetch internal transactions:\n"], - fn {entry, index}, acc -> - [acc, " ", to_string(index + 1), ". ", inspect(entry), "\n"] - end - ) - end) - - unique_entries - else - entries - end - end - - defp uniques_and_duplicates(groups) do - Enum.reduce(groups, {[], []}, fn group, {acc_uniques, acc_duplicates} -> - case group do - [unique] -> - {[unique | acc_uniques], acc_duplicates} - - [unique | _] = duplicates -> - {[unique | acc_uniques], duplicates ++ acc_duplicates} - end - end) - end - defp remove_failed_creations(internal_transactions_params) do internal_transactions_params |> Enum.map(fn internal_transaction_params -> diff --git a/apps/indexer/test/indexer/block/fetcher_test.exs b/apps/indexer/test/indexer/block/fetcher_test.exs index 3b7dd4bcff..4bd04dc6dc 100644 --- a/apps/indexer/test/indexer/block/fetcher_test.exs +++ b/apps/indexer/test/indexer/block/fetcher_test.exs @@ -485,8 +485,7 @@ defmodule Indexer.Block.FetcherTest do bytes: <<76, 188, 236, 37, 153, 153, 224, 115, 252, 79, 176, 224, 228, 166, 18, 66, 94, 61, 115, 57, 47, 162, 37, 255, 36, 96, 161, 238, 171, 66, 99, 10>> - }, - internal_transactions_indexed_at: nil + } }, %Transaction{ block_number: block_number, @@ -496,8 +495,7 @@ defmodule Indexer.Block.FetcherTest do bytes: <<240, 237, 34, 44, 16, 174, 248, 135, 4, 196, 15, 198, 34, 220, 218, 174, 13, 208, 242, 122, 154, 143, 4, 28, 171, 95, 190, 255, 254, 174, 75, 182>> - }, - internal_transactions_indexed_at: nil + } } ] }} = Fetcher.fetch_and_import_range(block_fetcher, block_number..block_number) @@ -589,8 +587,7 @@ defmodule Indexer.Block.FetcherTest do bytes: <<83, 189, 136, 72, 114, 222, 62, 72, 134, 146, 136, 27, 174, 236, 38, 46, 123, 149, 35, 77, 57, 101, 36, 140, 57, 254, 153, 47, 255, 212, 51, 229>> - }, - internal_transactions_indexed_at: nil + } } ] }, diff --git a/apps/indexer/test/indexer/fetcher/internal_transaction_test.exs b/apps/indexer/test/indexer/fetcher/internal_transaction_test.exs index 4807234df6..0486f7c8eb 100644 --- a/apps/indexer/test/indexer/fetcher/internal_transaction_test.exs +++ b/apps/indexer/test/indexer/fetcher/internal_transaction_test.exs @@ -2,10 +2,10 @@ defmodule Indexer.Fetcher.InternalTransactionTest do use EthereumJSONRPC.Case, async: false use Explorer.DataCase - import ExUnit.CaptureLog import Mox - alias Explorer.Chain.{Address, Hash, Transaction} + alias Explorer.Chain + alias Explorer.Chain.PendingBlockOperation alias Indexer.Fetcher.{CoinBalance, InternalTransaction, PendingTransaction} # MUST use global mode because we aren't guaranteed to get PendingTransactionFetcher's pid back fast enough to `allow` @@ -99,7 +99,8 @@ defmodule Indexer.Fetcher.InternalTransactionTest do end block_number = 1_000_006 - insert(:block, number: block_number) + block = insert(:block, number: block_number) + insert(:pending_block_operation, block_hash: block.hash, fetch_internal_transactions: true) assert :ok = InternalTransaction.run([block_number], json_rpc_named_arguments) @@ -111,54 +112,11 @@ defmodule Indexer.Fetcher.InternalTransactionTest do end describe "init/2" do - test "does not buffer pending transactions", %{json_rpc_named_arguments: json_rpc_named_arguments} do - insert(:transaction) - - assert InternalTransaction.init( - [], - fn hash_string, acc -> [hash_string | acc] end, - json_rpc_named_arguments - ) == [] - end - - @tag :no_parity - test "buffers collated transactions with unfetched internal transactions", %{ - json_rpc_named_arguments: json_rpc_named_arguments - } do - block = insert(:block) - - collated_unfetched_transaction = - :transaction - |> insert() - |> with_block(block) - - assert InternalTransaction.init( - [], - fn hash_string, acc -> [hash_string | acc] end, - json_rpc_named_arguments - ) == [{block.number, collated_unfetched_transaction.hash.bytes, collated_unfetched_transaction.index}] - end - - @tag :no_parity - test "does not buffer collated transactions with fetched internal transactions", %{ - json_rpc_named_arguments: json_rpc_named_arguments - } do - :transaction - |> insert() - |> with_block(internal_transactions_indexed_at: DateTime.utc_now()) - - assert InternalTransaction.init( - [], - fn hash_string, acc -> [hash_string | acc] end, - json_rpc_named_arguments - ) == [] - end - - @tag :no_geth test "buffers blocks with unfetched internal transactions", %{ json_rpc_named_arguments: json_rpc_named_arguments } do block = insert(:block) + insert(:pending_block_operation, block_hash: block.hash, fetch_internal_transactions: true) assert InternalTransaction.init( [], @@ -171,7 +129,8 @@ defmodule Indexer.Fetcher.InternalTransactionTest do test "does not buffer blocks with fetched internal transactions", %{ json_rpc_named_arguments: json_rpc_named_arguments } do - insert(:block, internal_transactions_indexed_at: DateTime.utc_now()) + block = insert(:block) + insert(:pending_block_operation, block_hash: block.hash, fetch_internal_transactions: false) assert InternalTransaction.init( [], @@ -182,135 +141,152 @@ defmodule Indexer.Fetcher.InternalTransactionTest do end describe "run/2" do - @tag :no_parity - test "duplicate transaction hashes are logged", %{json_rpc_named_arguments: json_rpc_named_arguments} do + test "handles empty block numbers", %{ + json_rpc_named_arguments: json_rpc_named_arguments + } do if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do - expect(EthereumJSONRPC.Mox, :json_rpc, fn _json, _options -> - {:ok, [%{id: 0, result: %{"trace" => []}}]} - end) + case Keyword.fetch!(json_rpc_named_arguments, :variant) do + EthereumJSONRPC.Parity -> + EthereumJSONRPC.Mox + |> expect(:json_rpc, fn [%{id: id}], _options -> + {:ok, + [ + %{ + id: id, + result: [] + } + ]} + end) + + EthereumJSONRPC.Geth -> + # do nothing, this block has no transactions, so Geth shouldn't query + :ok + + variant_name -> + raise ArgumentError, "Unsupported variant name (#{variant_name})" + end end - CoinBalance.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) + block = insert(:block) + block_hash = block.hash + insert(:pending_block_operation, block_hash: block_hash, fetch_internal_transactions: true) - %Transaction{hash: %Hash{bytes: bytes}} = - insert(:transaction, hash: "0x03cd5899a63b6f6222afda8705d059fd5a7d126bcabe962fb654d9736e6bcafa") - - log = - capture_log(fn -> - InternalTransaction.run( - [ - {1, bytes, 0}, - {1, bytes, 0} - ], - json_rpc_named_arguments - ) - end) + assert %{block_hash: block_hash} = Repo.get(PendingBlockOperation, block_hash) + + assert :ok == InternalTransaction.run([block.number], json_rpc_named_arguments) - assert log =~ - """ - Duplicate entries being used to fetch internal transactions: - 1. {1, <<3, 205, 88, 153, 166, 59, 111, 98, 34, 175, 218, 135, 5, 208, 89, 253, 90, 125, 18, 107, 202, 190, 150, 47, 182, 84, 217, 115, 110, 107, 202, 250>>, 0} - 2. {1, <<3, 205, 88, 153, 166, 59, 111, 98, 34, 175, 218, 135, 5, 208, 89, 253, 90, 125, 18, 107, 202, 190, 150, 47, 182, 84, 217, 115, 110, 107, 202, 250>>, 0} - """ + assert nil == Repo.get(PendingBlockOperation, block_hash) end - @tag :no_parity - test "internal transactions with failed parent does not create a new address", %{ + test "handles blocks with transactions correctly", %{ json_rpc_named_arguments: json_rpc_named_arguments } do + block = insert(:block) + transaction = insert(:transaction) |> with_block(block) + block_hash = block.hash + insert(:pending_block_operation, block_hash: block_hash, fetch_internal_transactions: true) + if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do - expect(EthereumJSONRPC.Mox, :json_rpc, fn _json, _options -> - {:ok, - [ - %{ - id: 0, - jsonrpc: "2.0", - result: %{ - "output" => "0x", - "stateDiff" => nil, - "trace" => [ - %{ - "action" => %{ + case Keyword.fetch!(json_rpc_named_arguments, :variant) do + EthereumJSONRPC.Parity -> + EthereumJSONRPC.Mox + |> expect(:json_rpc, fn [%{id: id, method: "trace_replayBlockTransactions"}], _options -> + {:ok, + [ + %{ + id: id, + result: [ + %{ + "output" => "0x", + "stateDiff" => nil, + "trace" => [ + %{ + "action" => %{ + "callType" => "call", + "from" => "0xa931c862e662134b85e4dc4baf5c70cc9ba74db4", + "gas" => "0x8600", + "input" => "0xb118e2db0000000000000000000000000000000000000000000000000000000000000008", + "to" => "0x1469b17ebf82fedf56f04109e5207bdc4554288c", + "value" => "0x174876e800" + }, + "result" => %{"gasUsed" => "0x7d37", "output" => "0x"}, + "subtraces" => 0, + "traceAddress" => [], + "type" => "call" + } + ], + "transactionHash" => transaction.hash, + "vmTrace" => nil + } + ] + } + ]} + end) + + EthereumJSONRPC.Geth -> + EthereumJSONRPC.Mox + |> expect(:json_rpc, fn [%{id: id, method: "debug_traceTransaction"}], _options -> + {:ok, + [ + %{ + id: id, + result: [ + %{ + "blockNumber" => block.number, + "transactionIndex" => 0, + "transactionHash" => transaction.hash, + "index" => 0, + "traceAddress" => [], + "type" => "call", "callType" => "call", - "from" => "0xc73add416e2119d20ce80e0904fc1877e33ef246", - "gas" => "0x13388", - "input" => "0xc793bf97", - "to" => "0x2d07e106b5d280e4ccc2d10deee62441c91d4340", - "value" => "0x0" - }, - "error" => "Reverted", - "subtraces" => 1, - "traceAddress" => [], - "type" => "call" - }, - %{ - "action" => %{ - "from" => "0x2d07e106b5d280e4ccc2d10deee62441c91d4340", - "gas" => "0xb2ab", - "init" => - "0x608060405234801561001057600080fd5b5060d38061001f6000396000f3fe6080604052600436106038577c010000000000000000000000000000000000000000000000000000000060003504633ccfd60b8114604f575b336000908152602081905260409020805434019055005b348015605a57600080fd5b5060616063565b005b33600081815260208190526040808220805490839055905190929183156108fc02918491818181858888f1935050505015801560a3573d6000803e3d6000fd5b505056fea165627a7a72305820e9a226f249def650de957dd8b4127b85a3049d6bfa818cadc4e2d3c44b6a53530029", - "value" => "0x0" - }, - "result" => %{ - "address" => "0xf4a5afe28b91cf928c2568805cfbb36d477f0b75", - "code" => - "0x6080604052600436106038577c010000000000000000000000000000000000000000000000000000000060003504633ccfd60b8114604f575b336000908152602081905260409020805434019055005b348015605a57600080fd5b5060616063565b005b33600081815260208190526040808220805490839055905190929183156108fc02918491818181858888f1935050505015801560a3573d6000803e3d6000fd5b505056fea165627a7a72305820e9a226f249def650de957dd8b4127b85a3049d6bfa818cadc4e2d3c44b6a53530029", - "gasUsed" => "0xa535" - }, - "subtraces" => 0, - "traceAddress" => [0], - "type" => "create" - } - ], - "vmTrace" => nil - } - } - ]} - end) - - CoinBalance.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) + "from" => "0xa931c862e662134b85e4dc4baf5c70cc9ba74db4", + "to" => "0x1469b17ebf82fedf56f04109e5207bdc4554288c", + "gas" => "0x8600", + "gasUsed" => "0x7d37", + "input" => "0xb118e2db0000000000000000000000000000000000000000000000000000000000000008", + "output" => "0x", + "value" => "0x174876e800" + } + ] + } + ]} + end) + + variant_name -> + raise ArgumentError, "Unsupported variant name (#{variant_name})" + end + end - %Transaction{hash: %Hash{bytes: bytes}} = - insert(:transaction, hash: "0x03cd5899a63b6f6222afda8705d059fd5a7d126bcabe962fb654d9736e6bcafa") - |> with_block() + CoinBalance.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) - :ok = - InternalTransaction.run( - [ - {7_202_692, bytes, 0} - ], - json_rpc_named_arguments - ) + assert %{block_hash: block_hash} = Repo.get(PendingBlockOperation, block_hash) - address = "0xf4a5afe28b91cf928c2568805cfbb36d477f0b75" + assert :ok == InternalTransaction.run([block.number], json_rpc_named_arguments) - fetched_address = Repo.one(from(address in Address, where: address.hash == ^address)) + assert nil == Repo.get(PendingBlockOperation, block_hash) - assert is_nil(fetched_address) - end + assert Repo.exists?(from(i in Chain.InternalTransaction, where: i.block_hash == ^block_hash)) end - @tag :no_parity - test "duplicate transaction hashes only retry uniques", %{json_rpc_named_arguments: json_rpc_named_arguments} do + test "handles failure by retrying only unique numbers", %{ + json_rpc_named_arguments: json_rpc_named_arguments + } do if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do expect(EthereumJSONRPC.Mox, :json_rpc, fn _json, _options -> {:ok, [%{id: 0, error: %{code: -32602, message: "Invalid params"}}]} end) end - CoinBalance.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) + block = insert(:block) + insert(:transaction) |> with_block(block) + block_hash = block.hash + insert(:pending_block_operation, block_hash: block_hash, fetch_internal_transactions: true) - # not a real transaction hash, so that fetch fails - %Transaction{hash: %Hash{bytes: bytes}} = - insert(:transaction, hash: "0x0000000000000000000000000000000000000000000000000000000000000001") + assert %{block_hash: block_hash} = Repo.get(PendingBlockOperation, block_hash) - assert InternalTransaction.run( - [ - {1, bytes, 0}, - {1, bytes, 0} - ], - json_rpc_named_arguments - ) == {:retry, [{1, bytes, 0}]} + assert {:retry, [block.number]} == InternalTransaction.run([block.number, block.number], json_rpc_named_arguments) + + assert %{block_hash: block_hash} = Repo.get(PendingBlockOperation, block_hash) end end end From 9f08c1c373f933641223a8b430fca38e58e34625 Mon Sep 17 00:00:00 2001 From: pasqu4le Date: Wed, 30 Oct 2019 19:03:10 +0100 Subject: [PATCH 17/57] test fix, comments and small refactorings --- .../address_read_contract_controller_test.exs | 6 ++- .../features/viewing_app_test.exs | 2 +- apps/explorer/lib/explorer/chain.ex | 44 +++++++------------ .../explorer/chain/import/runner/blocks.ex | 1 - .../import/runner/internal_transactions.ex | 38 ++++++++++------ .../chain/import/runner/transactions.ex | 6 +-- .../chain/import/stage/block_pending.ex | 5 ++- .../explorer/chain/internal_transaction.ex | 19 ++++---- .../indexer/fetcher/internal_transaction.ex | 3 +- 9 files changed, 60 insertions(+), 64 deletions(-) diff --git a/apps/block_scout_web/test/block_scout_web/controllers/address_read_contract_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/address_read_contract_controller_test.exs index ac3e39822b..4147053de1 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/address_read_contract_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/address_read_contract_controller_test.exs @@ -44,13 +44,15 @@ defmodule BlockScoutWeb.AddressReadContractControllerTest do test "returns not found for an unverified contract", %{conn: conn} do contract_address = insert(:contract_address) - transaction = insert(:transaction, from_address: contract_address) + transaction = insert(:transaction, from_address: contract_address) |> with_block() insert( :internal_transaction_create, index: 0, transaction: transaction, - created_contract_address: contract_address + created_contract_address: contract_address, + block_hash: transaction.block_hash, + block_index: 0 ) conn = get(conn, address_read_contract_path(BlockScoutWeb.Endpoint, :index, contract_address.hash)) diff --git a/apps/block_scout_web/test/block_scout_web/features/viewing_app_test.exs b/apps/block_scout_web/test/block_scout_web/features/viewing_app_test.exs index d7d7777d16..7fcb31893d 100644 --- a/apps/block_scout_web/test/block_scout_web/features/viewing_app_test.exs +++ b/apps/block_scout_web/test/block_scout_web/features/viewing_app_test.exs @@ -7,7 +7,7 @@ defmodule BlockScoutWeb.ViewingAppTest do alias BlockScoutWeb.Counters.BlocksIndexedCounter alias Explorer.Counters.AddressesCounter alias Explorer.{Repo} - alias Explorer.Chain.{PendingBlockOperation, Transaction} + alias Explorer.Chain.PendingBlockOperation setup do start_supervised!(AddressesCounter) diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index 74b910a711..1394d451e2 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -724,29 +724,20 @@ defmodule Explorer.Chain do """ @spec finished_indexing?() :: boolean() def finished_indexing? do - transaction_exists = - Transaction - |> limit(1) - |> Repo.one() - - min_block_number_transaction = Repo.aggregate(Transaction, :min, :block_number) - - if transaction_exists do - if min_block_number_transaction do - query = - from( - b in Block, - join: pending_ops in assoc(b, :pending_operations), - where: pending_ops.fetch_internal_transactions, - where: b.consensus and b.number == ^min_block_number_transaction - ) + with {:transactions_exist, true} <- {:transactions_exist, Repo.exists?(Transaction)}, + min_block_number when not is_nil(min_block_number) <- Repo.aggregate(Transaction, :min, :block_number) do + query = + from( + b in Block, + join: pending_ops in assoc(b, :pending_operations), + where: pending_ops.fetch_internal_transactions, + where: b.consensus and b.number == ^min_block_number + ) - !Repo.exists?(query) - else - false - end + !Repo.exists?(query) else - true + {:transactions_exist, false} -> true + nil -> false end end @@ -1338,10 +1329,7 @@ defmodule Explorer.Chain do @doc """ The number of `t:Explorer.Chain.InternalTransaction.t/0`. - iex> transaction = - ...> :transaction |> - ...> insert() |> - ...> with_block() + iex> transaction = :transaction |> insert() |> with_block() iex> insert(:internal_transaction, index: 0, transaction: transaction, block_hash: transaction.block_hash, block_index: 0) iex> Explorer.Chain.internal_transaction_count() 1 @@ -1353,8 +1341,7 @@ defmodule Explorer.Chain do """ def internal_transaction_count do - InternalTransaction.where_nonpending_block() - |> Repo.aggregate(:count, :transaction_hash) + Repo.aggregate(InternalTransaction.where_nonpending_block(), :count, :transaction_hash) end @doc """ @@ -1630,7 +1617,8 @@ defmodule Explorer.Chain do end @doc """ - Returns a stream of all blocks with unfetched internal transactions. + Returns a stream of all blocks with unfetched internal transactions, using + the `pending_block_operation` table. Only blocks with consensus are returned. diff --git a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex index 872e13ae7b..56373cbf6f 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex @@ -317,7 +317,6 @@ defmodule Explorer.Chain.Import.Runner.Blocks do nonconsensus_hashes |> MapSet.new() |> MapSet.union(MapSet.new(hashes)) - |> MapSet.to_list() |> Enum.sort() |> Enum.map(fn hash -> %{block_hash: hash, fetch_internal_transactions: true} diff --git a/apps/explorer/lib/explorer/chain/import/runner/internal_transactions.ex b/apps/explorer/lib/explorer/chain/import/runner/internal_transactions.ex index 38ec79b5a9..5c53e5fe6a 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/internal_transactions.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/internal_transactions.ex @@ -18,10 +18,7 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactions do # milliseconds @timeout 60_000 - @type imported :: [ - %{required(:index) => non_neg_integer(), required(:transaction_hash) => Hash.Full.t()} - | %{required(:empty_block_number) => non_neg_integer()} - ] + @type imported :: [InternalTransaction.t()] @impl Runner def ecto_schema_module, do: InternalTransaction @@ -50,7 +47,8 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactions do update_transactions_options = %{timeout: transactions_timeout, timestamps: timestamps} - internal_transactions_changes = Enum.filter(changes_list, &Map.has_key?(&1, :type)) + # filter out params with just `block_number` (indicating blocks without internal transactions) + internal_transactions_params = Enum.filter(changes_list, &Map.has_key?(&1, :type)) # Enforce ShareLocks tables order (see docs: sharelocks.md) multi @@ -64,14 +62,14 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactions do acquire_transactions(repo, pending_block_hashes) end) |> Multi.run(:invalid_block_numbers, fn _, %{acquire_transactions: transactions} -> - invalid_block_numbers(transactions, internal_transactions_changes) + invalid_block_numbers(transactions, internal_transactions_params) end) |> Multi.run(:valid_internal_transactions, fn _, %{ acquire_transactions: transactions, invalid_block_numbers: invalid_block_numbers } -> - valid_internal_transactions(transactions, internal_transactions_changes, invalid_block_numbers) + valid_internal_transactions(transactions, internal_transactions_params, invalid_block_numbers) end) |> Multi.run(:internal_transactions, fn repo, %{valid_internal_transactions: valid_internal_transactions} -> insert(repo, valid_internal_transactions, insert_options) @@ -201,6 +199,7 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactions do from( pending_ops in PendingBlockOperation, where: pending_ops.block_hash in ^block_hashes, + where: pending_ops.fetch_internal_transactions, select: pending_ops.block_hash, # Enforce PendingBlockOperation ShareLocks order (see docs: sharelocks.md) order_by: [asc: pending_ops.block_hash], @@ -224,10 +223,17 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactions do {:ok, repo.all(query)} end - defp invalid_block_numbers(transactions, internal_transactions_changes) do + defp invalid_block_numbers(transactions, internal_transactions_params) do + # Finds all mistmatches between transactions and internal transactions + # for a block number: + # - there are no internal txs for some transactions + # - there are no transactions for some internal transactions + # - there are internal txs with a different block number than their transactions + # Returns block numbers where any of these issues is found + required_tuples = MapSet.new(transactions, &{&1.hash, &1.block_number}) - candidate_tuples = MapSet.new(internal_transactions_changes, &{&1.transaction_hash, &1.block_number}) + candidate_tuples = MapSet.new(internal_transactions_params, &{&1.transaction_hash, &1.block_number}) all_tuples = MapSet.union(required_tuples, candidate_tuples) @@ -242,11 +248,11 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactions do {:ok, invalid_numbers} end - defp valid_internal_transactions(transactions, internal_transactions_changes, invalid_block_numbers) do + defp valid_internal_transactions(transactions, internal_transactions_params, invalid_block_numbers) do blocks_map = Map.new(transactions, &{&1.block_number, &1.block_hash}) valid_internal_txs = - internal_transactions_changes + internal_transactions_params |> Enum.group_by(& &1.block_number) |> Map.drop(invalid_block_numbers) |> Enum.flat_map(fn {block_number, entries} -> @@ -266,13 +272,17 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactions do end def defer_internal_transactions_primary_key(repo) do - # allows internal_transactions primary key to skips being checked during the - # DB transactions and instead be checked at the end of it. - # This allows us to use a more efficient upserting logic + # Allows internal_transactions primary key to not be checked during the + # DB transactions and instead be checked only at the end of it. + # This allows us to use a more efficient upserting logic, while keeping the + # uniqueness valid. SQL.query(repo, "SET CONSTRAINTS internal_transactions_pkey DEFERRED") end def remove_left_over_internal_transactions(repo, valid_internal_transactions) do + # Removes internal transactions that were part of a block before a refetch + # and have not been upserted with new ones (if any exist). + case valid_internal_transactions do [] -> {:ok, []} diff --git a/apps/explorer/lib/explorer/chain/import/runner/transactions.ex b/apps/explorer/lib/explorer/chain/import/runner/transactions.ex index 7a7265d1ae..db0a6ea968 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/transactions.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/transactions.ex @@ -78,10 +78,8 @@ defmodule Explorer.Chain.Import.Runner.Transactions do when is_list(changes_list) do on_conflict = Map.get_lazy(options, :on_conflict, &default_on_conflict/0) - ordered_changes_list = - changes_list - # Enforce Transaction ShareLocks order (see docs: sharelocks.md) - |> Enum.sort_by(& &1.hash) + # Enforce Transaction ShareLocks order (see docs: sharelocks.md) + ordered_changes_list = Enum.sort_by(changes_list, & &1.hash) Import.insert_changes_list( repo, diff --git a/apps/explorer/lib/explorer/chain/import/stage/block_pending.ex b/apps/explorer/lib/explorer/chain/import/stage/block_pending.ex index ef8088296d..fba315e142 100644 --- a/apps/explorer/lib/explorer/chain/import/stage/block_pending.ex +++ b/apps/explorer/lib/explorer/chain/import/stage/block_pending.ex @@ -1,7 +1,8 @@ defmodule Explorer.Chain.Import.Stage.BlockPending do @moduledoc """ - Imports any tables that follows and cannot be imported at the same time as - those imported by `Explorer.Chain.Import.Stage.Addresses`, + Imports any tables that uses `Explorer.Chain.PendingBlockOperation` to track + progress and cannot be imported at the same time as those imported by + `Explorer.Chain.Import.Stage.Addresses`, `Explorer.Chain.Import.Stage.AddressReferencing` and `Explorer.Chain.Import.Stage.BlockReferencing` """ diff --git a/apps/explorer/lib/explorer/chain/internal_transaction.ex b/apps/explorer/lib/explorer/chain/internal_transaction.ex index cd6c7996fd..c88cf11908 100644 --- a/apps/explorer/lib/explorer/chain/internal_transaction.ex +++ b/apps/explorer/lib/explorer/chain/internal_transaction.ex @@ -22,11 +22,15 @@ defmodule Explorer.Chain.InternalTransaction do * `to_address` - the sink of the `value` * `to_address_hash` - hash of the sink of the `value` * `trace_address` - list of traces - * `transaction` - transaction in which this transaction occurred + * `transaction` - transaction in which this internal transaction occurred * `transaction_hash` - foreign key for `transaction` * `transaction_index` - the `t:Explorer.Chain.Transaction.t/0` `index` of `transaction` in `block_number`. * `type` - type of internal transaction * `value` - value of transferred from `from_address` to `to_address` + * `block` - block in which this internal transaction occurred + * `block_hash` - foreign key for `block` + * `block_index` - the index of this internal transaction inside the `block` + * `pending_block` - `nil` if `block` has all its internal transactions fetched """ @type t :: %__MODULE__{ block_number: Explorer.Chain.Block.block_number() | nil, @@ -409,17 +413,12 @@ defmodule Explorer.Chain.InternalTransaction do on its own or use empty types to know that a block has no internal transactions. """ def blockless_changeset(%__MODULE__{} = internal_transaction, attrs \\ %{}) do - changeset = - internal_transaction - |> cast(attrs, ~w(type block_number)a) + changeset = cast(internal_transaction, attrs, ~w(type block_number)a) - if is_nil(get_field(changeset, :type)) do - changeset - |> validate_required(~w(block_number)a) + if validate_required(changeset, ~w(type)a).valid? do + type_changeset(changeset, attrs) else - changeset - |> validate_required(~w(type)a) - |> type_changeset(attrs) + validate_required(changeset, ~w(block_number)a) end end diff --git a/apps/indexer/lib/indexer/fetcher/internal_transaction.ex b/apps/indexer/lib/indexer/fetcher/internal_transaction.ex index db93c8d7e4..af7bdee23d 100644 --- a/apps/indexer/lib/indexer/fetcher/internal_transaction.ex +++ b/apps/indexer/lib/indexer/fetcher/internal_transaction.ex @@ -145,7 +145,6 @@ defmodule Indexer.Fetcher.InternalTransaction do end defp import_internal_transaction(internal_transactions_params, unique_numbers) do - unique_numbers_count = Enum.count(unique_numbers) internal_transactions_params_without_failed_creations = remove_failed_creations(internal_transactions_params) addresses_params = @@ -192,7 +191,7 @@ defmodule Indexer.Fetcher.InternalTransaction do ] end, step: step, - error_count: unique_numbers_count + error_count: Enum.count(unique_numbers) ) # re-queue the de-duped entries From 16763bff3f6ce7a467123f25f680fa638592cc41 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Wed, 20 Nov 2019 11:44:42 +0300 Subject: [PATCH 18/57] remove old internal transactions with the same primary key --- .../import/runner/internal_transactions.ex | 20 +++++++++----- .../runner/internal_transactions_test.exs | 27 +++++++++++++++++++ 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/apps/explorer/lib/explorer/chain/import/runner/internal_transactions.ex b/apps/explorer/lib/explorer/chain/import/runner/internal_transactions.ex index 5c53e5fe6a..69899f7584 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/internal_transactions.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/internal_transactions.ex @@ -71,13 +71,13 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactions do } -> valid_internal_transactions(transactions, internal_transactions_params, invalid_block_numbers) end) - |> Multi.run(:internal_transactions, fn repo, %{valid_internal_transactions: valid_internal_transactions} -> - insert(repo, valid_internal_transactions, insert_options) - end) |> Multi.run(:remove_left_over_internal_transactions, fn repo, %{valid_internal_transactions: valid_internal_transactions} -> remove_left_over_internal_transactions(repo, valid_internal_transactions) end) + |> Multi.run(:internal_transactions, fn repo, %{valid_internal_transactions: valid_internal_transactions} -> + insert(repo, valid_internal_transactions, insert_options) + end) |> Multi.run(:update_transactions, fn repo, %{valid_internal_transactions: valid_internal_transactions} -> update_transactions(repo, valid_internal_transactions, update_transactions_options) end) @@ -289,7 +289,7 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactions do _ -> try do - delete_query = + delete_query_for_block_hash_block_index = valid_internal_transactions |> Enum.group_by(& &1.block_hash, & &1.block_index) |> Enum.map(fn {block_hash, indexes} -> {block_hash, Enum.max(indexes)} end) @@ -297,10 +297,18 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactions do or_where(acc, [it], it.block_hash == ^block_hash and it.block_index > ^max_index) end) + # removes old recoreds with the same primary key (transaction hash, transaction index) + delete_query = + valid_internal_transactions + |> Enum.map(fn params -> {params.transaction_hash, params.index} end) + |> Enum.reduce(delete_query_for_block_hash_block_index, fn {transaction_hash, index}, acc -> + or_where(acc, [it], it.transaction_hash == ^transaction_hash and it.index == ^index) + end) + # ShareLocks order already enforced by `acquire_pending_internal_txs` (see docs: sharelocks.md) - {_count, result} = repo.delete_all(delete_query, []) + {count, result} = repo.delete_all(delete_query, []) - {:ok, result} + {:ok, {count, result}} rescue postgrex_error in Postgrex.Error -> {:error, %{exception: postgrex_error}} end diff --git a/apps/explorer/test/explorer/chain/import/runner/internal_transactions_test.exs b/apps/explorer/test/explorer/chain/import/runner/internal_transactions_test.exs index 96d54ca194..bb4ecc1ac1 100644 --- a/apps/explorer/test/explorer/chain/import/runner/internal_transactions_test.exs +++ b/apps/explorer/test/explorer/chain/import/runner/internal_transactions_test.exs @@ -85,6 +85,33 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactionsTest do assert PendingBlockOperation |> Repo.get(full_block.hash) |> is_nil() end + test "removes old records with the same primary key (transaction_hash, index)" do + full_block = insert(:block) + another_full_block = insert(:block) + + transaction = insert(:transaction) |> with_block(full_block) + + insert(:internal_transaction, + index: 0, + transaction: transaction, + block_hash: another_full_block.hash, + block_index: 0 + ) + + insert(:pending_block_operation, block_hash: full_block.hash, fetch_internal_transactions: true) + + transaction_changes = make_internal_transaction_changes(transaction, 0, nil) + + assert {:ok, %{remove_left_over_internal_transactions: {1, nil}}} = + run_internal_transactions([transaction_changes]) + + assert from(i in InternalTransaction, + where: i.transaction_hash == ^transaction.hash and i.block_hash == ^another_full_block.hash + ) + |> Repo.one() + |> is_nil() + end + test "removes consensus to blocks where not all transactions are filled" do full_block = insert(:block) transaction_a = insert(:transaction) |> with_block(full_block) From 6c90670534a6ceb9f81b19dc686fe13797cc7630 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Wed, 20 Nov 2019 12:45:07 +0300 Subject: [PATCH 19/57] add block_hash, block_index to primary key --- .../import/runner/internal_transactions.ex | 20 +++++--------- ...054_add_pending_internal_txs_operation.exs | 2 +- .../runner/internal_transactions_test.exs | 27 ------------------- 3 files changed, 7 insertions(+), 42 deletions(-) diff --git a/apps/explorer/lib/explorer/chain/import/runner/internal_transactions.ex b/apps/explorer/lib/explorer/chain/import/runner/internal_transactions.ex index 69899f7584..5c53e5fe6a 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/internal_transactions.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/internal_transactions.ex @@ -71,13 +71,13 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactions do } -> valid_internal_transactions(transactions, internal_transactions_params, invalid_block_numbers) end) + |> Multi.run(:internal_transactions, fn repo, %{valid_internal_transactions: valid_internal_transactions} -> + insert(repo, valid_internal_transactions, insert_options) + end) |> Multi.run(:remove_left_over_internal_transactions, fn repo, %{valid_internal_transactions: valid_internal_transactions} -> remove_left_over_internal_transactions(repo, valid_internal_transactions) end) - |> Multi.run(:internal_transactions, fn repo, %{valid_internal_transactions: valid_internal_transactions} -> - insert(repo, valid_internal_transactions, insert_options) - end) |> Multi.run(:update_transactions, fn repo, %{valid_internal_transactions: valid_internal_transactions} -> update_transactions(repo, valid_internal_transactions, update_transactions_options) end) @@ -289,7 +289,7 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactions do _ -> try do - delete_query_for_block_hash_block_index = + delete_query = valid_internal_transactions |> Enum.group_by(& &1.block_hash, & &1.block_index) |> Enum.map(fn {block_hash, indexes} -> {block_hash, Enum.max(indexes)} end) @@ -297,18 +297,10 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactions do or_where(acc, [it], it.block_hash == ^block_hash and it.block_index > ^max_index) end) - # removes old recoreds with the same primary key (transaction hash, transaction index) - delete_query = - valid_internal_transactions - |> Enum.map(fn params -> {params.transaction_hash, params.index} end) - |> Enum.reduce(delete_query_for_block_hash_block_index, fn {transaction_hash, index}, acc -> - or_where(acc, [it], it.transaction_hash == ^transaction_hash and it.index == ^index) - end) - # ShareLocks order already enforced by `acquire_pending_internal_txs` (see docs: sharelocks.md) - {count, result} = repo.delete_all(delete_query, []) + {_count, result} = repo.delete_all(delete_query, []) - {:ok, {count, result}} + {:ok, result} rescue postgrex_error in Postgrex.Error -> {:error, %{exception: postgrex_error}} end diff --git a/apps/explorer/priv/repo/migrations/20191018140054_add_pending_internal_txs_operation.exs b/apps/explorer/priv/repo/migrations/20191018140054_add_pending_internal_txs_operation.exs index 73493e832c..03c40ac2d2 100644 --- a/apps/explorer/priv/repo/migrations/20191018140054_add_pending_internal_txs_operation.exs +++ b/apps/explorer/priv/repo/migrations/20191018140054_add_pending_internal_txs_operation.exs @@ -57,7 +57,7 @@ defmodule Explorer.Repo.Migrations.AddPendingInternalTxsOperation do execute(""" ALTER table internal_transactions DROP CONSTRAINT internal_transactions_pkey, - ADD PRIMARY KEY (transaction_hash, index) DEFERRABLE; + ADD PRIMARY KEY (block_hash, block_index) DEFERRABLE; """) alter table(:internal_transactions) do diff --git a/apps/explorer/test/explorer/chain/import/runner/internal_transactions_test.exs b/apps/explorer/test/explorer/chain/import/runner/internal_transactions_test.exs index bb4ecc1ac1..96d54ca194 100644 --- a/apps/explorer/test/explorer/chain/import/runner/internal_transactions_test.exs +++ b/apps/explorer/test/explorer/chain/import/runner/internal_transactions_test.exs @@ -85,33 +85,6 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactionsTest do assert PendingBlockOperation |> Repo.get(full_block.hash) |> is_nil() end - test "removes old records with the same primary key (transaction_hash, index)" do - full_block = insert(:block) - another_full_block = insert(:block) - - transaction = insert(:transaction) |> with_block(full_block) - - insert(:internal_transaction, - index: 0, - transaction: transaction, - block_hash: another_full_block.hash, - block_index: 0 - ) - - insert(:pending_block_operation, block_hash: full_block.hash, fetch_internal_transactions: true) - - transaction_changes = make_internal_transaction_changes(transaction, 0, nil) - - assert {:ok, %{remove_left_over_internal_transactions: {1, nil}}} = - run_internal_transactions([transaction_changes]) - - assert from(i in InternalTransaction, - where: i.transaction_hash == ^transaction.hash and i.block_hash == ^another_full_block.hash - ) - |> Repo.one() - |> is_nil() - end - test "removes consensus to blocks where not all transactions are filled" do full_block = insert(:block) transaction_a = insert(:transaction) |> with_block(full_block) From 6953c71c89140f8253eff10f3b24dd9c3de42b63 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Wed, 20 Nov 2019 13:23:25 +0300 Subject: [PATCH 20/57] remove unique index --- .../20191018140054_add_pending_internal_txs_operation.exs | 2 -- 1 file changed, 2 deletions(-) diff --git a/apps/explorer/priv/repo/migrations/20191018140054_add_pending_internal_txs_operation.exs b/apps/explorer/priv/repo/migrations/20191018140054_add_pending_internal_txs_operation.exs index 03c40ac2d2..cc2d2b1800 100644 --- a/apps/explorer/priv/repo/migrations/20191018140054_add_pending_internal_txs_operation.exs +++ b/apps/explorer/priv/repo/migrations/20191018140054_add_pending_internal_txs_operation.exs @@ -52,8 +52,6 @@ defmodule Explorer.Repo.Migrations.AddPendingInternalTxsOperation do ; """) - create(unique_index(:internal_transactions, [:block_hash, :block_index])) - execute(""" ALTER table internal_transactions DROP CONSTRAINT internal_transactions_pkey, From ad5c8a42f2d2ecf378ef10f8d22cd33ad775fe7e Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Wed, 20 Nov 2019 13:29:39 +0300 Subject: [PATCH 21/57] remove deferrable --- .../20191018140054_add_pending_internal_txs_operation.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/explorer/priv/repo/migrations/20191018140054_add_pending_internal_txs_operation.exs b/apps/explorer/priv/repo/migrations/20191018140054_add_pending_internal_txs_operation.exs index cc2d2b1800..a12fa48836 100644 --- a/apps/explorer/priv/repo/migrations/20191018140054_add_pending_internal_txs_operation.exs +++ b/apps/explorer/priv/repo/migrations/20191018140054_add_pending_internal_txs_operation.exs @@ -55,7 +55,7 @@ defmodule Explorer.Repo.Migrations.AddPendingInternalTxsOperation do execute(""" ALTER table internal_transactions DROP CONSTRAINT internal_transactions_pkey, - ADD PRIMARY KEY (block_hash, block_index) DEFERRABLE; + ADD PRIMARY KEY (block_hash, block_index); """) alter table(:internal_transactions) do From 2a6b7e6dc1cea1cab835641b1713ebb5ee6f674c Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Thu, 21 Nov 2019 08:18:58 +0300 Subject: [PATCH 22/57] fix node_test.exs --- .../test/block_scout_web/schema/query/node_test.exs | 2 +- apps/explorer/test/support/factory.ex | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/block_scout_web/test/block_scout_web/schema/query/node_test.exs b/apps/block_scout_web/test/block_scout_web/schema/query/node_test.exs index 95acf02a61..4638e2f1c3 100644 --- a/apps/block_scout_web/test/block_scout_web/schema/query/node_test.exs +++ b/apps/block_scout_web/test/block_scout_web/schema/query/node_test.exs @@ -102,7 +102,7 @@ defmodule BlockScoutWeb.Schema.Query.NodeTest do end test "with 'id' for non-existent internal transaction", %{conn: conn} do - transaction = build(:transaction) |> with_block() + transaction = insert(:transaction) |> with_block() internal_transaction = build(:internal_transaction, diff --git a/apps/explorer/test/support/factory.ex b/apps/explorer/test/support/factory.ex index b26815c1f9..16166423a1 100644 --- a/apps/explorer/test/support/factory.ex +++ b/apps/explorer/test/support/factory.ex @@ -236,6 +236,8 @@ defmodule Explorer.Factory do block_hash: block_hash, block_number: block_number, cumulative_gas_used: cumulative_gas_used, + from_address_hash: transaction.from_address_hash, + to_address_hash: transaction.to_address_hash, error: error, gas_used: gas_used, index: next_transaction_index, From e667591560d21462b86546c19129855f0c304376 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Mon, 25 Nov 2019 06:16:18 +0300 Subject: [PATCH 23/57] fix tests --- .../test/block_scout_web/features/viewing_app_test.exs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/apps/block_scout_web/test/block_scout_web/features/viewing_app_test.exs b/apps/block_scout_web/test/block_scout_web/features/viewing_app_test.exs index 7fcb31893d..cac505ad66 100644 --- a/apps/block_scout_web/test/block_scout_web/features/viewing_app_test.exs +++ b/apps/block_scout_web/test/block_scout_web/features/viewing_app_test.exs @@ -29,6 +29,8 @@ defmodule BlockScoutWeb.ViewingAppTest do assert Decimal.cmp(Explorer.Chain.indexed_ratio(), Decimal.from_float(0.5)) == :eq + insert(:pending_block_operation, block_hash: block.hash, fetch_internal_transactions: true) + session |> AppPage.visit_page() |> assert_has(AppPage.indexed_status("50% Blocks Indexed")) @@ -46,6 +48,8 @@ defmodule BlockScoutWeb.ViewingAppTest do assert Decimal.cmp(Explorer.Chain.indexed_ratio(), 1) == :eq + insert(:pending_block_operation, block_hash: block.hash, fetch_internal_transactions: true) + session |> AppPage.visit_page() |> assert_has(AppPage.indexed_status("Indexing Tokens")) @@ -65,6 +69,8 @@ defmodule BlockScoutWeb.ViewingAppTest do assert Decimal.cmp(Explorer.Chain.indexed_ratio(), Decimal.from_float(0.5)) == :eq + insert(:pending_block_operation, block_hash: block.hash, fetch_internal_transactions: true) + session |> AppPage.visit_page() |> assert_has(AppPage.indexed_status("50% Blocks Indexed")) @@ -90,6 +96,8 @@ defmodule BlockScoutWeb.ViewingAppTest do assert Decimal.cmp(Explorer.Chain.indexed_ratio(), Decimal.from_float(0.9)) == :eq + insert(:pending_block_operation, block_hash: block.hash, fetch_internal_transactions: true) + session |> AppPage.visit_page() |> assert_has(AppPage.indexed_status("90% Blocks Indexed")) From 2bd6f66e99ae5976e6d8d04009406ad7832f9be4 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Mon, 25 Nov 2019 06:38:00 +0300 Subject: [PATCH 24/57] remove nonconsensus blocks from pending blocks --- apps/explorer/lib/explorer/chain.ex | 16 ++++++++++++++++ apps/explorer/test/explorer/chain_test.exs | 16 ++++++++++++++++ .../lib/indexer/fetcher/internal_transaction.ex | 2 ++ 3 files changed, 34 insertions(+) diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index 1394d451e2..304d146d48 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -38,6 +38,7 @@ defmodule Explorer.Chain do Import, InternalTransaction, Log, + PendingBlockOperation, SmartContract, StakingPool, Token, @@ -1660,6 +1661,21 @@ defmodule Explorer.Chain do Repo.stream_reduce(query, initial, reducer) end + @spec remove_nonconsensus_blocks_from_pending_ops() :: :ok + def remove_nonconsensus_blocks_from_pending_ops do + query = + from( + po in PendingBlockOperation, + inner_join: block in Block, + on: block.hash == po.block_hash, + where: block.consensus == false + ) + + {_, _} = Repo.delete_all(query) + + :ok + end + @spec stream_transactions_with_unfetched_created_contract_codes( fields :: [ :block_hash diff --git a/apps/explorer/test/explorer/chain_test.exs b/apps/explorer/test/explorer/chain_test.exs index e67a6ec196..ac8e48e48e 100644 --- a/apps/explorer/test/explorer/chain_test.exs +++ b/apps/explorer/test/explorer/chain_test.exs @@ -18,6 +18,7 @@ defmodule Explorer.ChainTest do Hash, InternalTransaction, Log, + PendingBlockOperation, Token, TokenTransfer, Transaction, @@ -35,6 +36,21 @@ defmodule Explorer.ChainTest do setup :verify_on_exit! + describe "remove_nonconsensus_blocks_from_pending_ops/0" do + test "removes pending ops for nonconsensus blocks" do + block = insert(:block) + insert(:pending_block_operation, block: block, fetch_internal_transactions: true) + + nonconsensus_block = insert(:block, consensus: false) + insert(:pending_block_operation, block: nonconsensus_block, fetch_internal_transactions: true) + + :ok = Chain.remove_nonconsensus_blocks_from_pending_ops() + + assert Repo.get(PendingBlockOperation, block.hash) + assert is_nil(Repo.get(PendingBlockOperation, nonconsensus_block.hash)) + end + end + describe "count_addresses_with_balance_from_cache/0" do test "returns the number of addresses with fetched_coin_balance > 0" do insert(:address, fetched_coin_balance: 0) diff --git a/apps/indexer/lib/indexer/fetcher/internal_transaction.ex b/apps/indexer/lib/indexer/fetcher/internal_transaction.ex index af7bdee23d..ea907129b1 100644 --- a/apps/indexer/lib/indexer/fetcher/internal_transaction.ex +++ b/apps/indexer/lib/indexer/fetcher/internal_transaction.ex @@ -76,6 +76,8 @@ defmodule Indexer.Fetcher.InternalTransaction do reducer.(block_number, acc) end) + :ok = Chain.remove_nonconsensus_blocks_from_pending_ops() + final end From 036c16fb09d82e35fad9d1e9a96cac8308ca0161 Mon Sep 17 00:00:00 2001 From: pasqu4le Date: Fri, 18 Oct 2019 15:59:33 +0200 Subject: [PATCH 25/57] Add pending block operations Problem: a lot of problems are created by tracking the importing progress on multiple tables. Querying and updating involves as such heavy operations and this is also the cause for inconsistencies. Solution: use a new table to track the progress of block's data indexing. By using a specific table, whose rows are deleted when there are no more pending operations on the relative block, we make joins quick and keep them to a minimum. This is also meant to simplify all inserts and updates and only perform deletes when strictly necessary. --- apps/explorer/lib/explorer/chain/pending_block_operation.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/explorer/lib/explorer/chain/pending_block_operation.ex b/apps/explorer/lib/explorer/chain/pending_block_operation.ex index d85cc589b0..df849ebe37 100644 --- a/apps/explorer/lib/explorer/chain/pending_block_operation.ex +++ b/apps/explorer/lib/explorer/chain/pending_block_operation.ex @@ -16,7 +16,7 @@ defmodule Explorer.Chain.PendingBlockOperation do @type t :: %__MODULE__{ block_hash: Hash.Full.t(), fetch_internal_transactions: boolean() - } + } @primary_key false schema "pending_block_operations" do From c0808ee6ce9d1e3867ba6ca85be80fbaf2a4f978 Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Mon, 4 Nov 2019 15:47:54 +0300 Subject: [PATCH 26/57] Update internal transactions constraint fixing script --- CHANGELOG.md | 1 + ...ional_internal_transaction_constraints.sql | 67 ++++++++++--------- 2 files changed, 38 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bf01641a5f..e446671578 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,7 @@ - [#2844](https://github.com/poanetwork/blockscout/pull/2844) - Extend external reward types up to 20 - [#2827](https://github.com/poanetwork/blockscout/pull/2827) - Node js 12.13.0 (latest LTS release) support - [#2818](https://github.com/poanetwork/blockscout/pull/2818) - allow hiding marketcap percentage +- [#2835](https://github.com/poanetwork/blockscout/pull/2835) - Update int txs constraints repairing script - [#2817](https://github.com/poanetwork/blockscout/pull/2817) - move docker integration documentation to blockscout docs - [#2808](https://github.com/poanetwork/blockscout/pull/2808) - Add tooltip for tx input - [#2807](https://github.com/poanetwork/blockscout/pull/2807) - 422 page diff --git a/apps/explorer/priv/repo/migrations/scripts/20181108205650_large_additional_internal_transaction_constraints.sql b/apps/explorer/priv/repo/migrations/scripts/20181108205650_large_additional_internal_transaction_constraints.sql index cb82ce8847..77c576e549 100644 --- a/apps/explorer/priv/repo/migrations/scripts/20181108205650_large_additional_internal_transaction_constraints.sql +++ b/apps/explorer/priv/repo/migrations/scripts/20181108205650_large_additional_internal_transaction_constraints.sql @@ -6,6 +6,7 @@ -- IMPORTANT NOTE: after making all the corrections needed the script will NOT -- run the constraint validations because this may be a very long and taxing -- operation. To validate the constraint one can run, after the script fininshed: +-- UPDATE (2019-11-04): use pending_block_operations table instead of internal_transactions -- ALTER TABLE internal_transactions VALIDATE CONSTRAINT call_has_call_type; -- ALTER TABLE internal_transactions VALIDATE CONSTRAINT call_has_input; @@ -14,56 +15,62 @@ DO $$ DECLARE batch_size integer := 10000; -- HOW MANY ITEMS WILL BE UPDATED AT A TIME - last_transaction_hash bytea; -- WILL CHECK ONLY TRANSACTIONS FOLLOWING THIS HASH (DESC) + last_block_number integer; -- WILL CHECK ONLY TRANSACTIONS FOLLOWING THIS HASH (DESC) last_fetched_batch_size integer; BEGIN RAISE NOTICE 'STARTING SCRIPT'; - CREATE TEMP TABLE transactions_with_deprecated_internal_transactions(hash bytea NOT NULL); + CREATE TEMP TABLE blocks_with_deprecated_internal_transactions(block_number integer NOT NULL); LOOP RAISE NOTICE 'Fetching new batch of % transactions to correct', batch_size; - INSERT INTO transactions_with_deprecated_internal_transactions - SELECT DISTINCT transaction_hash - FROM internal_transactions - WHERE - (last_transaction_hash IS NULL OR transaction_hash < last_transaction_hash) AND - -- call_has_call_type CONSTRAINT - ((type = 'call' AND call_type IS NULL) OR - -- call_has_input CONSTRAINT - (type = 'call' AND input IS NULL) OR - -- create_has_init CONSTRAINT - (type = 'create' AND init is NULL)) - ORDER BY transaction_hash DESC LIMIT batch_size; + INSERT INTO blocks_with_deprecated_internal_transactions + SELECT DISTINCT a.block_number + FROM ( + SELECT DISTINCT i.block_number, i.transaction_index + FROM internal_transactions i + WHERE + i.block_number IS NOT NULL + AND + (last_block_number IS NULL OR i.block_number < last_block_number) AND + -- call_has_call_type CONSTRAINT + ((type = 'call' AND i.call_type IS NULL) OR + -- call_has_input CONSTRAINT + (type = 'call' AND i.input IS NULL) OR + -- create_has_init CONSTRAINT + (type = 'create' AND i.init is NULL)) + ORDER BY i.block_number DESC, i.transaction_index LIMIT batch_size + ) a; - SELECT INTO last_fetched_batch_size count(*) FROM transactions_with_deprecated_internal_transactions; + SELECT INTO last_fetched_batch_size count(block_number) FROM blocks_with_deprecated_internal_transactions; RAISE NOTICE 'Batch of % transactions was fetched, starting their deprecation', last_fetched_batch_size; - -- UPDATE TRANSACTIONS - UPDATE transactions - SET internal_transactions_indexed_at = NULL, - error = NULL - FROM transactions_with_deprecated_internal_transactions - WHERE transactions.hash = transactions_with_deprecated_internal_transactions.hash; + INSERT INTO pending_block_operations (block_hash, inserted_at, updated_at, fetch_internal_transactions) + SELECT b.hash, NOW(), NOW(), true + FROM blocks_with_deprecated_internal_transactions bd, blocks b + WHERE bd.block_number = b.number + AND b.consensus = true + ON CONFLICT (block_hash) + DO NOTHING; -- REMOVE THE DEPRECATED internal_transactions DELETE FROM internal_transactions - USING transactions_with_deprecated_internal_transactions - WHERE internal_transactions.transaction_hash = transactions_with_deprecated_internal_transactions.hash; + USING blocks_with_deprecated_internal_transactions + WHERE internal_transactions.block_number = blocks_with_deprecated_internal_transactions.block_number; -- COMMIT THE BATCH UPDATES CHECKPOINT; - -- UPDATE last_transaction_hash TO KEEP TRACK OF ROWS ALREADY CHECKED - SELECT INTO last_transaction_hash hash - FROM transactions_with_deprecated_internal_transactions - ORDER BY hash ASC LIMIT 1; + -- UPDATE last_block_number TO KEEP TRACK OF ROWS ALREADY CHECKED + SELECT INTO last_block_number block_number + FROM blocks_with_deprecated_internal_transactions + ORDER BY block_number ASC LIMIT 1; - RAISE NOTICE 'Last batch completed, last transaction hash: %', last_transaction_hash; + RAISE NOTICE 'Last batch completed, last block number: %', last_block_number; -- CLEAR THE TEMP TABLE - DELETE FROM transactions_with_deprecated_internal_transactions; + DELETE FROM blocks_with_deprecated_internal_transactions; -- EXIT IF ALL internal_transactions HAVE BEEN CHECKED ALREADY EXIT WHEN last_fetched_batch_size != batch_size; @@ -71,5 +78,5 @@ BEGIN RAISE NOTICE 'SCRIPT FINISHED, all affected transactions have been deprecated'; - DROP TABLE transactions_with_deprecated_internal_transactions; + DROP TABLE blocks_with_deprecated_internal_transactions; END $$; From 03c744542a932233b3f2e88d8ac99b951cf7d866 Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Sun, 17 Nov 2019 20:19:50 +0300 Subject: [PATCH 27/57] type to i.type --- ...50_large_additional_internal_transaction_constraints.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/explorer/priv/repo/migrations/scripts/20181108205650_large_additional_internal_transaction_constraints.sql b/apps/explorer/priv/repo/migrations/scripts/20181108205650_large_additional_internal_transaction_constraints.sql index 77c576e549..669ac06da7 100644 --- a/apps/explorer/priv/repo/migrations/scripts/20181108205650_large_additional_internal_transaction_constraints.sql +++ b/apps/explorer/priv/repo/migrations/scripts/20181108205650_large_additional_internal_transaction_constraints.sql @@ -34,11 +34,11 @@ BEGIN AND (last_block_number IS NULL OR i.block_number < last_block_number) AND -- call_has_call_type CONSTRAINT - ((type = 'call' AND i.call_type IS NULL) OR + ((i.type = 'call' AND i.call_type IS NULL) OR -- call_has_input CONSTRAINT - (type = 'call' AND i.input IS NULL) OR + (i.type = 'call' AND i.input IS NULL) OR -- create_has_init CONSTRAINT - (type = 'create' AND i.init is NULL)) + (i.type = 'create' AND i.init is NULL)) ORDER BY i.block_number DESC, i.transaction_index LIMIT batch_size ) a; From 16514a8f10a0a2b2671e30f15ab14373df187c84 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Wed, 20 Nov 2019 11:44:42 +0300 Subject: [PATCH 28/57] remove old internal transactions with the same primary key --- .../import/runner/internal_transactions.ex | 20 +++++++++----- .../runner/internal_transactions_test.exs | 27 +++++++++++++++++++ 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/apps/explorer/lib/explorer/chain/import/runner/internal_transactions.ex b/apps/explorer/lib/explorer/chain/import/runner/internal_transactions.ex index 5c53e5fe6a..69899f7584 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/internal_transactions.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/internal_transactions.ex @@ -71,13 +71,13 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactions do } -> valid_internal_transactions(transactions, internal_transactions_params, invalid_block_numbers) end) - |> Multi.run(:internal_transactions, fn repo, %{valid_internal_transactions: valid_internal_transactions} -> - insert(repo, valid_internal_transactions, insert_options) - end) |> Multi.run(:remove_left_over_internal_transactions, fn repo, %{valid_internal_transactions: valid_internal_transactions} -> remove_left_over_internal_transactions(repo, valid_internal_transactions) end) + |> Multi.run(:internal_transactions, fn repo, %{valid_internal_transactions: valid_internal_transactions} -> + insert(repo, valid_internal_transactions, insert_options) + end) |> Multi.run(:update_transactions, fn repo, %{valid_internal_transactions: valid_internal_transactions} -> update_transactions(repo, valid_internal_transactions, update_transactions_options) end) @@ -289,7 +289,7 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactions do _ -> try do - delete_query = + delete_query_for_block_hash_block_index = valid_internal_transactions |> Enum.group_by(& &1.block_hash, & &1.block_index) |> Enum.map(fn {block_hash, indexes} -> {block_hash, Enum.max(indexes)} end) @@ -297,10 +297,18 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactions do or_where(acc, [it], it.block_hash == ^block_hash and it.block_index > ^max_index) end) + # removes old recoreds with the same primary key (transaction hash, transaction index) + delete_query = + valid_internal_transactions + |> Enum.map(fn params -> {params.transaction_hash, params.index} end) + |> Enum.reduce(delete_query_for_block_hash_block_index, fn {transaction_hash, index}, acc -> + or_where(acc, [it], it.transaction_hash == ^transaction_hash and it.index == ^index) + end) + # ShareLocks order already enforced by `acquire_pending_internal_txs` (see docs: sharelocks.md) - {_count, result} = repo.delete_all(delete_query, []) + {count, result} = repo.delete_all(delete_query, []) - {:ok, result} + {:ok, {count, result}} rescue postgrex_error in Postgrex.Error -> {:error, %{exception: postgrex_error}} end diff --git a/apps/explorer/test/explorer/chain/import/runner/internal_transactions_test.exs b/apps/explorer/test/explorer/chain/import/runner/internal_transactions_test.exs index 96d54ca194..bb4ecc1ac1 100644 --- a/apps/explorer/test/explorer/chain/import/runner/internal_transactions_test.exs +++ b/apps/explorer/test/explorer/chain/import/runner/internal_transactions_test.exs @@ -85,6 +85,33 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactionsTest do assert PendingBlockOperation |> Repo.get(full_block.hash) |> is_nil() end + test "removes old records with the same primary key (transaction_hash, index)" do + full_block = insert(:block) + another_full_block = insert(:block) + + transaction = insert(:transaction) |> with_block(full_block) + + insert(:internal_transaction, + index: 0, + transaction: transaction, + block_hash: another_full_block.hash, + block_index: 0 + ) + + insert(:pending_block_operation, block_hash: full_block.hash, fetch_internal_transactions: true) + + transaction_changes = make_internal_transaction_changes(transaction, 0, nil) + + assert {:ok, %{remove_left_over_internal_transactions: {1, nil}}} = + run_internal_transactions([transaction_changes]) + + assert from(i in InternalTransaction, + where: i.transaction_hash == ^transaction.hash and i.block_hash == ^another_full_block.hash + ) + |> Repo.one() + |> is_nil() + end + test "removes consensus to blocks where not all transactions are filled" do full_block = insert(:block) transaction_a = insert(:transaction) |> with_block(full_block) From b92ad5e9ce6de1f51b8da4c39c8c78cc46cd989a Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Thu, 28 Nov 2019 16:32:19 +0300 Subject: [PATCH 29/57] fix merging conflicts --- apps/explorer/lib/explorer/chain/pending_block_operation.ex | 2 +- apps/indexer/lib/indexer/fetcher/internal_transaction.ex | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/explorer/lib/explorer/chain/pending_block_operation.ex b/apps/explorer/lib/explorer/chain/pending_block_operation.ex index df849ebe37..d85cc589b0 100644 --- a/apps/explorer/lib/explorer/chain/pending_block_operation.ex +++ b/apps/explorer/lib/explorer/chain/pending_block_operation.ex @@ -16,7 +16,7 @@ defmodule Explorer.Chain.PendingBlockOperation do @type t :: %__MODULE__{ block_hash: Hash.Full.t(), fetch_internal_transactions: boolean() - } + } @primary_key false schema "pending_block_operations" do diff --git a/apps/indexer/lib/indexer/fetcher/internal_transaction.ex b/apps/indexer/lib/indexer/fetcher/internal_transaction.ex index ea907129b1..ea7fda1397 100644 --- a/apps/indexer/lib/indexer/fetcher/internal_transaction.ex +++ b/apps/indexer/lib/indexer/fetcher/internal_transaction.ex @@ -13,7 +13,7 @@ defmodule Indexer.Fetcher.InternalTransaction do import Indexer.Block.Fetcher, only: [async_import_coin_balances: 2] alias Explorer.Chain - alias Explorer.Chain.{Block, Hash} + alias Explorer.Chain.Block alias Explorer.Chain.Cache.{Accounts, Blocks} alias Indexer.{BufferedTask, Tracer} alias Indexer.Transform.Addresses From 00f12015cc4ad3efe56bfecb572741deafbbafa7 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Wed, 27 Nov 2019 18:39:10 +0300 Subject: [PATCH 30/57] remove pending ops on block invalidation --- apps/explorer/lib/explorer/chain.ex | 13 ++++++++++++- .../lib/explorer/chain/import/runner/blocks.ex | 4 +++- apps/explorer/test/explorer/chain_test.exs | 17 +++++++++++++++++ .../lib/indexer/fetcher/internal_transaction.ex | 2 -- 4 files changed, 32 insertions(+), 4 deletions(-) diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index 304d146d48..e15efe8114 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -1661,7 +1661,18 @@ defmodule Explorer.Chain do Repo.stream_reduce(query, initial, reducer) end - @spec remove_nonconsensus_blocks_from_pending_ops() :: :ok + def remove_nonconsensus_blocks_from_pending_ops(block_hashes) do + query = + from( + po in PendingBlockOperation, + where: po.block_hash in ^block_hashes + ) + + {_, _} = Repo.delete_all(query) + + :ok + end + def remove_nonconsensus_blocks_from_pending_ops do query = from( diff --git a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex index 56373cbf6f..7944d37187 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex @@ -8,7 +8,7 @@ defmodule Explorer.Chain.Import.Runner.Blocks do import Ecto.Query, only: [from: 2, subquery: 1] alias Ecto.{Changeset, Multi, Repo} - + alias Explorer.Chain alias Explorer.Chain.{Address, Block, Import, Log, PendingBlockOperation, Transaction} alias Explorer.Chain.Block.Reward alias Explorer.Chain.Import.Runner @@ -306,6 +306,8 @@ defmodule Explorer.Chain.Import.Runner.Blocks do timeout: timeout ) + :ok = Chain.remove_nonconsensus_blocks_from_pending_ops(removed_consensus_block_hashes) + {:ok, removed_consensus_block_hashes} rescue postgrex_error in Postgrex.Error -> diff --git a/apps/explorer/test/explorer/chain_test.exs b/apps/explorer/test/explorer/chain_test.exs index ac8e48e48e..a8b95bb6d3 100644 --- a/apps/explorer/test/explorer/chain_test.exs +++ b/apps/explorer/test/explorer/chain_test.exs @@ -49,6 +49,23 @@ defmodule Explorer.ChainTest do assert Repo.get(PendingBlockOperation, block.hash) assert is_nil(Repo.get(PendingBlockOperation, nonconsensus_block.hash)) end + + test "removes pending ops for nonconsensus blocks by block hashes" do + block = insert(:block) + insert(:pending_block_operation, block: block, fetch_internal_transactions: true) + + nonconsensus_block = insert(:block, consensus: false) + insert(:pending_block_operation, block: nonconsensus_block, fetch_internal_transactions: true) + + nonconsensus_block1 = insert(:block, consensus: false) + insert(:pending_block_operation, block: nonconsensus_block1, fetch_internal_transactions: true) + + :ok = Chain.remove_nonconsensus_blocks_from_pending_ops([nonconsensus_block1.hash]) + + assert Repo.get(PendingBlockOperation, block.hash) + assert Repo.get(PendingBlockOperation, nonconsensus_block.hash) + assert is_nil(Repo.get(PendingBlockOperation, nonconsensus_block1.hash)) + end end describe "count_addresses_with_balance_from_cache/0" do diff --git a/apps/indexer/lib/indexer/fetcher/internal_transaction.ex b/apps/indexer/lib/indexer/fetcher/internal_transaction.ex index ea7fda1397..aa9a967cd1 100644 --- a/apps/indexer/lib/indexer/fetcher/internal_transaction.ex +++ b/apps/indexer/lib/indexer/fetcher/internal_transaction.ex @@ -76,8 +76,6 @@ defmodule Indexer.Fetcher.InternalTransaction do reducer.(block_number, acc) end) - :ok = Chain.remove_nonconsensus_blocks_from_pending_ops() - final end From a2a32115fd691b01207ca7393096daae75c5ae41 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Thu, 21 Nov 2019 10:18:33 +0300 Subject: [PATCH 31/57] do not remove non consensus logs --- .../explorer/chain/import/runner/blocks.ex | 32 ------------------ ...add_block_hash_and_block_index_to_logs.exs | 33 +++++++++++++++++++ 2 files changed, 33 insertions(+), 32 deletions(-) create mode 100644 apps/explorer/priv/repo/migrations/20191121064805_add_block_hash_and_block_index_to_logs.exs diff --git a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex index 56373cbf6f..444c5a7241 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex @@ -87,9 +87,6 @@ defmodule Explorer.Chain.Import.Runner.Blocks do transactions: transactions }) end) - |> Multi.run(:remove_nonconsensus_logs, fn repo, %{derive_transaction_forks: transactions} -> - remove_nonconsensus_logs(repo, transactions, insert_options) - end) |> Multi.run(:acquire_contract_address_tokens, fn repo, _ -> acquire_contract_address_tokens(repo, consensus_block_numbers) end) @@ -334,35 +331,6 @@ defmodule Explorer.Chain.Import.Runner.Blocks do ) end - defp remove_nonconsensus_logs(repo, forked_transaction_hashes, %{timeout: timeout}) do - ordered_logs = - from( - log in Log, - where: log.transaction_hash in ^forked_transaction_hashes, - select: log.transaction_hash, - # Enforce Log ShareLocks order (see docs: sharelocks.md) - order_by: [ - log.transaction_hash, - log.index - ], - lock: "FOR UPDATE" - ) - - query = - from(log in Log, - select: map(log, [:transaction_hash, :index]), - inner_join: ordered_log in subquery(ordered_logs), - on: ordered_log.transaction_hash == log.transaction_hash - ) - - {_count, deleted_logs} = repo.delete_all(query, timeout: timeout) - - {:ok, deleted_logs} - rescue - postgrex_error in Postgrex.Error -> - {:error, %{exception: postgrex_error, transactions: forked_transaction_hashes}} - end - defp delete_address_token_balances(_, [], _), do: {:ok, []} defp delete_address_token_balances(repo, consensus_block_numbers, %{timeout: timeout}) do diff --git a/apps/explorer/priv/repo/migrations/20191121064805_add_block_hash_and_block_index_to_logs.exs b/apps/explorer/priv/repo/migrations/20191121064805_add_block_hash_and_block_index_to_logs.exs new file mode 100644 index 0000000000..3b4882f9cb --- /dev/null +++ b/apps/explorer/priv/repo/migrations/20191121064805_add_block_hash_and_block_index_to_logs.exs @@ -0,0 +1,33 @@ +defmodule Explorer.Repo.Migrations.AddBlockHashAndBlockIndexToLogs do + use Ecto.Migration + + def change do + alter table(:logs) do + add(:block_hash, :bytea) + end + + execute(""" + UPDATE logs log + SET block_hash = with_block.block_hash + FROM ( + SELECT l.transaction_hash, + t.block_hash + FROM logs l + JOIN transactions t + ON t.hash = l.transaction_hash + ) AS with_block + WHERE log.transaction_hash = with_block.transaction_hash + ; + """) + + alter table(:logs) do + modify(:block_hash, references(:blocks, column: :hash, type: :bytea), null: false) + end + + execute(""" + ALTER table logs + DROP CONSTRAINT logs_pkey, + ADD PRIMARY KEY (transaction_hash, block_hash, index); + """) + end +end From 1f108d767329b38f6c5178cc6ea13437d48b9b41 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Thu, 21 Nov 2019 10:46:25 +0300 Subject: [PATCH 32/57] set block_hash in logs params --- apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/log.ex | 1 + .../lib/explorer/chain/import/runner/blocks.ex | 2 +- .../lib/explorer/chain/import/runner/logs.ex | 2 +- apps/explorer/lib/explorer/chain/log.ex | 14 ++++++++++++-- 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/log.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/log.ex index 52cdae96bd..7eb0ee5bd7 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/log.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/log.ex @@ -98,6 +98,7 @@ defmodule EthereumJSONRPC.Log do block_number: block_number, block_hash: block_hash, data: data, + block_hash: block_hash, index: index, transaction_hash: transaction_hash } diff --git a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex index 444c5a7241..36b426e9ca 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex @@ -9,7 +9,7 @@ defmodule Explorer.Chain.Import.Runner.Blocks do alias Ecto.{Changeset, Multi, Repo} - alias Explorer.Chain.{Address, Block, Import, Log, PendingBlockOperation, Transaction} + alias Explorer.Chain.{Address, Block, Import, PendingBlockOperation, Transaction} alias Explorer.Chain.Block.Reward alias Explorer.Chain.Import.Runner alias Explorer.Chain.Import.Runner.Address.CurrentTokenBalances diff --git a/apps/explorer/lib/explorer/chain/import/runner/logs.ex b/apps/explorer/lib/explorer/chain/import/runner/logs.ex index ddac92a647..0cc8a92b71 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/logs.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/logs.ex @@ -65,7 +65,7 @@ defmodule Explorer.Chain.Import.Runner.Logs do Import.insert_changes_list( repo, ordered_changes_list, - conflict_target: [:transaction_hash, :index], + conflict_target: [:transaction_hash, :index, :block_hash], on_conflict: on_conflict, for: Log, returning: true, diff --git a/apps/explorer/lib/explorer/chain/log.ex b/apps/explorer/lib/explorer/chain/log.ex index 7916b56fb3..7c4daf9692 100644 --- a/apps/explorer/lib/explorer/chain/log.ex +++ b/apps/explorer/lib/explorer/chain/log.ex @@ -6,14 +6,15 @@ defmodule Explorer.Chain.Log do require Logger alias ABI.{Event, FunctionSelector} - alias Explorer.Chain.{Address, ContractMethod, Data, Hash, Transaction} + alias Explorer.Chain.{Address, Block, ContractMethod, Data, Hash, Transaction} alias Explorer.Repo - @required_attrs ~w(address_hash data index transaction_hash)a + @required_attrs ~w(address_hash data block_hash index transaction_hash)a @optional_attrs ~w(first_topic second_topic third_topic fourth_topic type)a @typedoc """ * `address` - address of contract that generate the event + * `block_hash` - hash of the block * `address_hash` - foreign key for `address` * `data` - non-indexed log parameters. * `first_topic` - `topics[0]` @@ -28,6 +29,7 @@ defmodule Explorer.Chain.Log do @type t :: %__MODULE__{ address: %Ecto.Association.NotLoaded{} | Address.t(), address_hash: Hash.Address.t(), + block_hash: Hash.Full.t(), data: Data.t(), first_topic: String.t(), second_topic: String.t(), @@ -59,6 +61,14 @@ defmodule Explorer.Chain.Log do references: :hash, type: Hash.Full ) + + + belongs_to(:block, Block, + foreign_key: :block_hash, + primary_key: true, + references: :hash, + type: Hash.Full + ) end @doc """ From 45610a88d298a174b2d03a4dd1eb2d436556427e Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Thu, 21 Nov 2019 11:06:45 +0300 Subject: [PATCH 33/57] fix chain tests --- apps/explorer/lib/explorer/chain/log.ex | 1 - apps/explorer/test/explorer/chain_test.exs | 1 + apps/explorer/test/support/factory.ex | 1 + 3 files changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/explorer/lib/explorer/chain/log.ex b/apps/explorer/lib/explorer/chain/log.ex index 7c4daf9692..d0ff825122 100644 --- a/apps/explorer/lib/explorer/chain/log.ex +++ b/apps/explorer/lib/explorer/chain/log.ex @@ -62,7 +62,6 @@ defmodule Explorer.Chain.Log do type: Hash.Full ) - belongs_to(:block, Block, foreign_key: :block_hash, primary_key: true, diff --git a/apps/explorer/test/explorer/chain_test.exs b/apps/explorer/test/explorer/chain_test.exs index ac8e48e48e..c9f121254b 100644 --- a/apps/explorer/test/explorer/chain_test.exs +++ b/apps/explorer/test/explorer/chain_test.exs @@ -1311,6 +1311,7 @@ defmodule Explorer.ChainTest do logs: %{ params: [ %{ + block_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd", address_hash: "0x8bf38d4764929064f2d4d3a56520a76ab3df415b", data: "0x0000000000000000000000000000000000000000000000000de0b6b3a7640000", first_topic: "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", diff --git a/apps/explorer/test/support/factory.ex b/apps/explorer/test/support/factory.ex index 16166423a1..191833d663 100644 --- a/apps/explorer/test/support/factory.ex +++ b/apps/explorer/test/support/factory.ex @@ -355,6 +355,7 @@ defmodule Explorer.Factory do def log_factory do %Log{ address: build(:address), + block: build(:block), data: data(:log_data), first_topic: nil, fourth_topic: nil, From cbd98a3e048a820d587d23869ce70c6c9092d78e Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Thu, 21 Nov 2019 12:25:20 +0300 Subject: [PATCH 34/57] fix most tests --- apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/log.ex | 2 ++ .../test/ethereum_jsonrpc/receipts_test.exs | 1 + apps/explorer/lib/explorer/chain/log.ex | 1 + apps/explorer/test/explorer/chain/import_test.exs | 9 ++++++++- apps/explorer/test/explorer/chain/log_test.exs | 10 ++++++++-- 5 files changed, 20 insertions(+), 3 deletions(-) diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/log.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/log.ex index 7eb0ee5bd7..11f91b1fec 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/log.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/log.ex @@ -48,6 +48,7 @@ defmodule EthereumJSONRPC.Log do second_topic: nil, third_topic: nil, transaction_hash: "0x53bd884872de3e488692881baeec262e7b95234d3965248c39fe992fffd433e5", + block_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd", type: "mined" } @@ -72,6 +73,7 @@ defmodule EthereumJSONRPC.Log do address_hash: "0xda8b3276cde6d768a44b9dac659faa339a41ac55", block_hash: "0x0b89f7f894f5d8ba941e16b61490e999a0fcaaf92dfcc70aee2ac5ddb5f243e1", block_number: 4448, + block_hash: "0x0b89f7f894f5d8ba941e16b61490e999a0fcaaf92dfcc70aee2ac5ddb5f243e1", data: "0x290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563", first_topic: "0xadc1e8a294f8415511303acc4a8c0c5906c7eb0bf2a71043d7f4b03b46a39130", fourth_topic: nil, diff --git a/apps/ethereum_jsonrpc/test/ethereum_jsonrpc/receipts_test.exs b/apps/ethereum_jsonrpc/test/ethereum_jsonrpc/receipts_test.exs index 468e0f0f30..dfa172850c 100644 --- a/apps/ethereum_jsonrpc/test/ethereum_jsonrpc/receipts_test.exs +++ b/apps/ethereum_jsonrpc/test/ethereum_jsonrpc/receipts_test.exs @@ -84,6 +84,7 @@ defmodule EthereumJSONRPC.ReceiptsTest do "logs" => [ %{ "address" => address_hash, + "blockHash" => "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd", "blockNumber" => integer_to_quantity(block_number), "blockHash" => nil, "data" => data, diff --git a/apps/explorer/lib/explorer/chain/log.ex b/apps/explorer/lib/explorer/chain/log.ex index d0ff825122..d197acb51d 100644 --- a/apps/explorer/lib/explorer/chain/log.ex +++ b/apps/explorer/lib/explorer/chain/log.ex @@ -78,6 +78,7 @@ defmodule Explorer.Chain.Log do ...> %Explorer.Chain.Log{}, ...> %{ ...> address_hash: "0x8bf38d4764929064f2d4d3a56520a76ab3df415b", + ...> block_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd", ...> data: "0x000000000000000000000000862d67cb0773ee3f8ce7ea89b328ffea861ab3ef", ...> first_topic: "0x600bcf04a13e752d1e3670a5a9f1c21177ca2a93c6f5391d4f1298d098097c22", ...> fourth_topic: nil, diff --git a/apps/explorer/test/explorer/chain/import_test.exs b/apps/explorer/test/explorer/chain/import_test.exs index 468801b77c..7a06fa2779 100644 --- a/apps/explorer/test/explorer/chain/import_test.exs +++ b/apps/explorer/test/explorer/chain/import_test.exs @@ -88,6 +88,7 @@ defmodule Explorer.Chain.ImportTest do logs: %{ params: [ %{ + block_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd", address_hash: "0x8bf38d4764929064f2d4d3a56520a76ab3df415b", data: "0x0000000000000000000000000000000000000000000000000de0b6b3a7640000", first_topic: "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", @@ -1518,7 +1519,13 @@ defmodule Explorer.Chain.ImportTest do with: :blockless_changeset }, logs: %{ - params: [params_for(:log, transaction_hash: transaction_hash, address_hash: miner_hash)], + params: [ + params_for(:log, + transaction_hash: transaction_hash, + address_hash: miner_hash, + block_hash: block_hash + ) + ], timeout: 1 }, token_transfers: %{ diff --git a/apps/explorer/test/explorer/chain/log_test.exs b/apps/explorer/test/explorer/chain/log_test.exs index e4ce0695bc..3619476f6c 100644 --- a/apps/explorer/test/explorer/chain/log_test.exs +++ b/apps/explorer/test/explorer/chain/log_test.exs @@ -9,7 +9,12 @@ defmodule Explorer.Chain.LogTest do describe "changeset/2" do test "accepts valid attributes" do - params = params_for(:log, address_hash: build(:address).hash, transaction_hash: build(:transaction).hash) + params = + params_for(:log, + address_hash: build(:address).hash, + transaction_hash: build(:transaction).hash, + block_hash: build(:block).hash + ) assert %Changeset{valid?: true} = Log.changeset(%Log{}, params) end @@ -26,7 +31,8 @@ defmodule Explorer.Chain.LogTest do :log, address_hash: build(:address).hash, first_topic: "ham", - transaction_hash: build(:transaction).hash + transaction_hash: build(:transaction).hash, + block_hash: build(:block).hash ) assert %Changeset{changes: %{first_topic: "ham"}, valid?: true} = Log.changeset(%Log{}, params) From b92ebeb0263aaeb60136a9600bca6c1a7468abb8 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Thu, 21 Nov 2019 12:40:39 +0300 Subject: [PATCH 35/57] fix remaining tests --- .../api/rpc/eth_controller_test.exs | 30 +++++++++++-------- .../api/rpc/transaction_controller_test.exs | 3 +- .../chain/import/runner/blocks_test.exs | 16 ---------- 3 files changed, 19 insertions(+), 30 deletions(-) diff --git a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/eth_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/eth_controller_test.exs index 5ab050aefc..68d2271c2b 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/eth_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/eth_controller_test.exs @@ -76,7 +76,7 @@ defmodule BlockScoutWeb.API.RPC.EthControllerTest do block = insert(:block, number: 0) transaction = insert(:transaction, from_address: address) |> with_block(block) - insert(:log, address: address, transaction: transaction, data: "0x010101") + insert(:log, block: block, address: address, transaction: transaction, data: "0x010101") params = params(api_params, [%{"address" => to_string(address.hash)}]) @@ -94,7 +94,7 @@ defmodule BlockScoutWeb.API.RPC.EthControllerTest do block = insert(:block, number: 0) transaction = insert(:transaction, from_address: address) |> with_block(block) - insert(:log, address: address, transaction: transaction, data: "0x010101", first_topic: "0x01") + insert(:log, block: block, address: address, transaction: transaction, data: "0x010101", first_topic: "0x01") params = params(api_params, [%{"address" => to_string(address.hash), "topics" => ["0x01"]}]) @@ -112,8 +112,8 @@ defmodule BlockScoutWeb.API.RPC.EthControllerTest do block = insert(:block, number: 0) transaction = insert(:transaction, from_address: address) |> with_block(block) - insert(:log, address: address, transaction: transaction, data: "0x010101", first_topic: "0x01") - insert(:log, address: address, transaction: transaction, data: "0x020202", first_topic: "0x00") + insert(:log, address: address, block: block, transaction: transaction, data: "0x010101", first_topic: "0x01") + insert(:log, address: address, block: block, transaction: transaction, data: "0x020202", first_topic: "0x00") params = params(api_params, [%{"address" => to_string(address.hash), "topics" => [["0x01", "0x00"]]}]) @@ -127,14 +127,15 @@ defmodule BlockScoutWeb.API.RPC.EthControllerTest do test "paginates logs", %{conn: conn, api_params: api_params} do contract_address = insert(:contract_address) + block = insert(:block) transaction = :transaction |> insert(to_address: contract_address) - |> with_block() + |> with_block(block) inserted_records = - insert_list(2000, :log, address: contract_address, transaction: transaction, first_topic: "0x01") + insert_list(2000, :log, block: block, address: contract_address, transaction: transaction, first_topic: "0x01") params = params(api_params, [%{"address" => to_string(contract_address), "topics" => [["0x01"]]}]) @@ -191,10 +192,11 @@ defmodule BlockScoutWeb.API.RPC.EthControllerTest do transaction: transaction, data: "0x010101", first_topic: "0x01", - second_topic: "0x02" + second_topic: "0x02", + block: block ) - insert(:log, address: address, transaction: transaction, data: "0x020202", first_topic: "0x01") + insert(:log, block: block, address: address, transaction: transaction, data: "0x020202", first_topic: "0x01") params = params(api_params, [%{"address" => to_string(address.hash), "topics" => ["0x01", "0x02"]}]) @@ -219,7 +221,8 @@ defmodule BlockScoutWeb.API.RPC.EthControllerTest do transaction: transaction, data: "0x010101", first_topic: "0x01", - second_topic: "0x02" + second_topic: "0x02", + block: block ) insert(:log, @@ -227,7 +230,8 @@ defmodule BlockScoutWeb.API.RPC.EthControllerTest do transaction: transaction, data: "0x020202", first_topic: "0x01", - second_topic: "0x03" + second_topic: "0x03", + block: block ) params = params(api_params, [%{"address" => to_string(address.hash), "topics" => ["0x01", ["0x02", "0x03"]]}]) @@ -341,11 +345,11 @@ defmodule BlockScoutWeb.API.RPC.EthControllerTest do transaction2 = insert(:transaction, from_address: address) |> with_block(block2) transaction3 = insert(:transaction, from_address: address) |> with_block(block3) - insert(:log, address: address, transaction: transaction1, data: "0x010101") + insert(:log, block: block1, address: address, transaction: transaction1, data: "0x010101") - insert(:log, address: address, transaction: transaction2, data: "0x020202") + insert(:log, block: block2, address: address, transaction: transaction2, data: "0x020202") - insert(:log, address: address, transaction: transaction3, data: "0x030303") + insert(:log, block: block3, address: address, transaction: transaction3, data: "0x030303") changeset = Ecto.Changeset.change(block3, %{consensus: false}) diff --git a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/transaction_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/transaction_controller_test.exs index d83e530c28..416ad729fc 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/transaction_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/transaction_controller_test.exs @@ -486,7 +486,8 @@ defmodule BlockScoutWeb.API.RPC.TransactionControllerTest do address: address, transaction: transaction, first_topic: "first topic", - second_topic: "second topic" + second_topic: "second topic", + block: block ) params = %{ diff --git a/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs b/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs index d6c278c1c8..4d724a673b 100644 --- a/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs +++ b/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs @@ -115,22 +115,6 @@ defmodule Explorer.Chain.Import.Runner.BlocksTest do assert count(Address.CurrentTokenBalance) == count end - test "remove_nonconsensus_logs deletes nonconsensus logs", %{ - consensus_block: %{number: block_number} = block, - options: options - } do - old_block = insert(:block, number: block_number, consensus: true) - forked_transaction = :transaction |> insert() |> with_block(old_block) - %Log{transaction_hash: hash, index: index} = insert(:log, transaction: forked_transaction) - - assert count(Log) == 1 - - assert {:ok, %{remove_nonconsensus_logs: [%{transaction_hash: ^hash, index: ^index}]}} = - run_block_consensus_change(block, true, options) - - assert count(Log) == 0 - end - test "derive_address_current_token_balances inserts rows if there is an address_token_balance left for the rows deleted by delete_address_current_token_balances", %{consensus_block: %{number: block_number} = block, options: options} do token = insert(:token) From 19007f930ed0d29002832a49742620b65cfdd4e1 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Thu, 21 Nov 2019 13:49:34 +0300 Subject: [PATCH 36/57] fix query --- apps/explorer/lib/explorer/chain.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index 304d146d48..a11a8d97ba 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -365,7 +365,7 @@ defmodule Explorer.Chain do or_where: transaction.block_number == ^block_number and transaction.index == ^transaction_index and log.index > ^log_index, - where: log.address_hash == ^address_hash, + where: log.address_hash == ^address_hash and log.block_hash == transaction.block_hash, limit: ^paging_options.page_size, select: log ) From 53ca6fd48f180ae4704651384b4806272f852c43 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Fri, 22 Nov 2019 08:57:46 +0300 Subject: [PATCH 37/57] fix chain_test --- apps/explorer/test/explorer/chain_test.exs | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/apps/explorer/test/explorer/chain_test.exs b/apps/explorer/test/explorer/chain_test.exs index c9f121254b..a8f0291ac6 100644 --- a/apps/explorer/test/explorer/chain_test.exs +++ b/apps/explorer/test/explorer/chain_test.exs @@ -236,14 +236,14 @@ defmodule Explorer.ChainTest do |> insert(to_address: address) |> with_block() - insert(:log, transaction: transaction1, index: 1, address: address) + insert(:log, block: transaction1.block, transaction: transaction1, index: 1, address: address) transaction2 = :transaction |> insert(from_address: address) |> with_block() - insert(:log, transaction: transaction2, index: 2, address: address) + insert(:log, block: transaction2.block, transaction: transaction2, index: 2, address: address) assert Enum.count(Chain.address_to_logs(address_hash)) == 2 end @@ -259,7 +259,9 @@ defmodule Explorer.ChainTest do log1 = insert(:log, transaction: transaction, index: 1, address: address) 2..51 - |> Enum.map(fn index -> insert(:log, transaction: transaction, index: index, address: address) end) + |> Enum.map(fn index -> + insert(:log, block: transaction.block, transaction: transaction, index: index, address: address) + end) |> Enum.map(& &1.index) paging_options1 = %PagingOptions{page_size: 1} @@ -279,14 +281,14 @@ defmodule Explorer.ChainTest do |> insert(to_address: address) |> with_block() - insert(:log, transaction: transaction1, index: 1, address: address) + insert(:log, block: transaction1.block, transaction: transaction1, index: 1, address: address) transaction2 = :transaction |> insert(from_address: address) |> with_block() - insert(:log, transaction: transaction2, index: 2, address: address, first_topic: "test") + insert(:log, block: transaction2.block, transaction: transaction2, index: 2, address: address, first_topic: "test") [found_log] = Chain.address_to_logs(address_hash, topic: "test") @@ -301,14 +303,20 @@ defmodule Explorer.ChainTest do |> insert(to_address: address) |> with_block() - insert(:log, transaction: transaction1, index: 1, address: address, fourth_topic: "test") + insert(:log, + block: transaction1.block, + transaction: transaction1, + index: 1, + address: address, + fourth_topic: "test" + ) transaction2 = :transaction |> insert(from_address: address) |> with_block() - insert(:log, transaction: transaction2, index: 2, address: address) + insert(:log, block: transaction2.block, transaction: transaction2, index: 2, address: address) [found_log] = Chain.address_to_logs(address_hash, topic: "test") From e109865ceb0a36c485a2ca806d7844f996efdd33 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Sat, 23 Nov 2019 09:47:07 +0300 Subject: [PATCH 38/57] add CHANGELOG entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e446671578..9c5081e222 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ - [#2829](https://github.com/poanetwork/blockscout/pull/2829) - Fix for stuck gas limit label and value - [#2828](https://github.com/poanetwork/blockscout/pull/2828) - Fix for script that clears compilation/launching assets - [#2872](https://github.com/poanetwork/blockscout/pull/2872) - do not remove token transfers +- [#2871](https://github.com/poanetwork/blockscout/pull/2871) - do not remove logs - [#2800](https://github.com/poanetwork/blockscout/pull/2800) - return not found for not verified contract for token read_contract - [#2806](https://github.com/poanetwork/blockscout/pull/2806) - Fix blocks fetching on the main page - [#2803](https://github.com/poanetwork/blockscout/pull/2803) - Fix block validator custom tooltip From daaa17b330a91e7a50219051eacb7b3e17bad0d2 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Sat, 23 Nov 2019 13:54:10 +0300 Subject: [PATCH 39/57] add block_hash to sorting --- apps/explorer/lib/explorer/chain/import/runner/logs.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/explorer/lib/explorer/chain/import/runner/logs.ex b/apps/explorer/lib/explorer/chain/import/runner/logs.ex index 0cc8a92b71..1a816c0a93 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/logs.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/logs.ex @@ -59,7 +59,7 @@ defmodule Explorer.Chain.Import.Runner.Logs do on_conflict = Map.get_lazy(options, :on_conflict, &default_on_conflict/0) # Enforce Log ShareLocks order (see docs: sharelocks.md) - ordered_changes_list = Enum.sort_by(changes_list, &{&1.transaction_hash, &1.index}) + ordered_changes_list = Enum.sort_by(changes_list, &{&1.transaction_hash, &1.block_hash, &1.index}) {:ok, _} = Import.insert_changes_list( From 380b34f4cb304adefc9f92132ac11899c874d5a9 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Wed, 27 Nov 2019 07:09:29 +0300 Subject: [PATCH 40/57] add block_number to logs --- apps/explorer/lib/explorer/chain/log.ex | 5 ++++- ...191121064805_add_block_hash_and_block_index_to_logs.exs | 7 +++++-- .../test/explorer/chain/import/runner/blocks_test.exs | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/apps/explorer/lib/explorer/chain/log.ex b/apps/explorer/lib/explorer/chain/log.ex index d197acb51d..27899303f3 100644 --- a/apps/explorer/lib/explorer/chain/log.ex +++ b/apps/explorer/lib/explorer/chain/log.ex @@ -10,11 +10,12 @@ defmodule Explorer.Chain.Log do alias Explorer.Repo @required_attrs ~w(address_hash data block_hash index transaction_hash)a - @optional_attrs ~w(first_topic second_topic third_topic fourth_topic type)a + @optional_attrs ~w(first_topic second_topic third_topic fourth_topic type block_number)a @typedoc """ * `address` - address of contract that generate the event * `block_hash` - hash of the block + * `block_number` - The block number that the transfer took place. * `address_hash` - foreign key for `address` * `data` - non-indexed log parameters. * `first_topic` - `topics[0]` @@ -30,6 +31,7 @@ defmodule Explorer.Chain.Log do address: %Ecto.Association.NotLoaded{} | Address.t(), address_hash: Hash.Address.t(), block_hash: Hash.Full.t(), + block_number: non_neg_integer() | nil, data: Data.t(), first_topic: String.t(), second_topic: String.t(), @@ -50,6 +52,7 @@ defmodule Explorer.Chain.Log do field(:fourth_topic, :string) field(:index, :integer, primary_key: true) field(:type, :string) + field(:block_number, :integer) timestamps() diff --git a/apps/explorer/priv/repo/migrations/20191121064805_add_block_hash_and_block_index_to_logs.exs b/apps/explorer/priv/repo/migrations/20191121064805_add_block_hash_and_block_index_to_logs.exs index 3b4882f9cb..3856339771 100644 --- a/apps/explorer/priv/repo/migrations/20191121064805_add_block_hash_and_block_index_to_logs.exs +++ b/apps/explorer/priv/repo/migrations/20191121064805_add_block_hash_and_block_index_to_logs.exs @@ -4,14 +4,17 @@ defmodule Explorer.Repo.Migrations.AddBlockHashAndBlockIndexToLogs do def change do alter table(:logs) do add(:block_hash, :bytea) + add(:block_number, :integer) end execute(""" UPDATE logs log - SET block_hash = with_block.block_hash + SET block_hash = with_block.block_hash, + block_number = with_block.block_number FROM ( SELECT l.transaction_hash, - t.block_hash + t.block_hash, + t.block_number FROM logs l JOIN transactions t ON t.hash = l.transaction_hash diff --git a/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs b/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs index 4d724a673b..02a62cab2b 100644 --- a/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs +++ b/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs @@ -7,7 +7,7 @@ defmodule Explorer.Chain.Import.Runner.BlocksTest do alias Ecto.Multi alias Explorer.Chain.Import.Runner.{Blocks, Transactions} - alias Explorer.Chain.{Address, Block, Log, Transaction, TokenTransfer} + alias Explorer.Chain.{Address, Block, Transaction, TokenTransfer} alias Explorer.{Chain, Repo} describe "run/1" do From e127fd124cae00750045d300f2c7ef257038e16f Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Wed, 27 Nov 2019 07:38:10 +0300 Subject: [PATCH 41/57] fix logs queries --- apps/explorer/lib/explorer/chain.ex | 16 +++++++++++++--- apps/explorer/test/support/factory.ex | 5 ++++- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index a11a8d97ba..5a52b2198b 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -357,7 +357,10 @@ defmodule Explorer.Chain do base_query = from(log in Log, - inner_join: transaction in assoc(log, :transaction), + inner_join: transaction in Transaction, + on: + transaction.block_hash == log.block_hash and transaction.block_number == log.block_number and + transaction.hash == log.transaction_hash, order_by: [desc: transaction.block_number, desc: transaction.index], preload: [:transaction, transaction: [to_address: :smart_contract]], where: transaction.block_number < ^block_number, @@ -2387,8 +2390,15 @@ defmodule Explorer.Chain do necessity_by_association = Keyword.get(options, :necessity_by_association, %{}) paging_options = Keyword.get(options, :paging_options, @default_paging_options) - Log - |> join(:inner, [log], transaction in assoc(log, :transaction)) + log_with_transactions = + from(log in Log, + inner_join: transaction in Transaction, + on: + transaction.block_hash == log.block_hash and transaction.block_number == log.block_number and + transaction.hash == log.transaction_hash + ) + + log_with_transactions |> where([_, transaction], transaction.hash == ^transaction_hash) |> page_logs(paging_options) |> limit(^paging_options.page_size) diff --git a/apps/explorer/test/support/factory.ex b/apps/explorer/test/support/factory.ex index 191833d663..668d5fa6f6 100644 --- a/apps/explorer/test/support/factory.ex +++ b/apps/explorer/test/support/factory.ex @@ -353,9 +353,12 @@ defmodule Explorer.Factory do end def log_factory do + block = build(:block) + %Log{ address: build(:address), - block: build(:block), + block: block, + block_number: block.number, data: data(:log_data), first_topic: nil, fourth_topic: nil, From 6226602eecffa2578fdd3d3cf9cd8a4621813969 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Wed, 27 Nov 2019 09:06:42 +0300 Subject: [PATCH 42/57] fix tests --- .../api/rpc/transaction_controller_test.exs | 7 +- .../transaction_log_controller_test.exs | 42 +++++++++-- apps/explorer/test/explorer/chain_test.exs | 74 ++++++++++++++++--- 3 files changed, 105 insertions(+), 18 deletions(-) diff --git a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/transaction_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/transaction_controller_test.exs index 416ad729fc..2b3f6fff5e 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/transaction_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/transaction_controller_test.exs @@ -411,7 +411,9 @@ defmodule BlockScoutWeb.API.RPC.TransactionControllerTest do address: address, transaction: transaction, first_topic: "first topic", - second_topic: "second topic" + second_topic: "second topic", + block: block, + block_number: block.number ) end) @@ -487,7 +489,8 @@ defmodule BlockScoutWeb.API.RPC.TransactionControllerTest do transaction: transaction, first_topic: "first topic", second_topic: "second topic", - block: block + block: block, + block_number: block.number ) params = %{ diff --git a/apps/block_scout_web/test/block_scout_web/controllers/transaction_log_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/transaction_log_controller_test.exs index 65142dd0ce..5e316f4483 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/transaction_log_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/transaction_log_controller_test.exs @@ -29,7 +29,13 @@ defmodule BlockScoutWeb.TransactionLogControllerTest do |> with_block() address = insert(:address) - insert(:log, address: address, transaction: transaction) + + insert(:log, + address: address, + transaction: transaction, + block: transaction.block, + block_number: transaction.block_number + ) conn = get(conn, transaction_log_path(conn, :index, transaction), %{type: "JSON"}) @@ -46,7 +52,13 @@ defmodule BlockScoutWeb.TransactionLogControllerTest do |> with_block() address = insert(:address) - insert(:log, address: address, transaction: transaction) + + insert(:log, + address: address, + transaction: transaction, + block: transaction.block, + block_number: transaction.block_number + ) conn = get(conn, transaction_log_path(conn, :index, transaction), %{type: "JSON"}) @@ -73,11 +85,24 @@ defmodule BlockScoutWeb.TransactionLogControllerTest do |> insert() |> with_block() - log = insert(:log, transaction: transaction, index: 1) + log = + insert(:log, + transaction: transaction, + index: 1, + block: transaction.block, + block_number: transaction.block_number + ) second_page_indexes = 2..51 - |> Enum.map(fn index -> insert(:log, transaction: transaction, index: index) end) + |> Enum.map(fn index -> + insert(:log, + transaction: transaction, + index: index, + block: transaction.block, + block_number: transaction.block_number + ) + end) |> Enum.map(& &1.index) conn = @@ -98,7 +123,14 @@ defmodule BlockScoutWeb.TransactionLogControllerTest do |> with_block() 1..60 - |> Enum.map(fn index -> insert(:log, transaction: transaction, index: index) end) + |> Enum.map(fn index -> + insert(:log, + transaction: transaction, + index: index, + block: transaction.block, + block_number: transaction.block_number + ) + end) conn = get(conn, transaction_log_path(conn, :index, transaction), %{type: "JSON"}) diff --git a/apps/explorer/test/explorer/chain_test.exs b/apps/explorer/test/explorer/chain_test.exs index a8f0291ac6..9256ad70d0 100644 --- a/apps/explorer/test/explorer/chain_test.exs +++ b/apps/explorer/test/explorer/chain_test.exs @@ -236,14 +236,26 @@ defmodule Explorer.ChainTest do |> insert(to_address: address) |> with_block() - insert(:log, block: transaction1.block, transaction: transaction1, index: 1, address: address) + insert(:log, + block: transaction1.block, + block_number: transaction1.block_number, + transaction: transaction1, + index: 1, + address: address + ) transaction2 = :transaction |> insert(from_address: address) |> with_block() - insert(:log, block: transaction2.block, transaction: transaction2, index: 2, address: address) + insert(:log, + block: transaction2.block, + block_number: transaction2.block_number, + transaction: transaction2, + index: 2, + address: address + ) assert Enum.count(Chain.address_to_logs(address_hash)) == 2 end @@ -256,11 +268,17 @@ defmodule Explorer.ChainTest do |> insert(to_address: address) |> with_block() - log1 = insert(:log, transaction: transaction, index: 1, address: address) + log1 = insert(:log, transaction: transaction, index: 1, address: address, block_number: transaction.block_number) 2..51 |> Enum.map(fn index -> - insert(:log, block: transaction.block, transaction: transaction, index: index, address: address) + insert(:log, + block: transaction.block, + transaction: transaction, + index: index, + address: address, + block_number: transaction.block_number + ) end) |> Enum.map(& &1.index) @@ -281,14 +299,27 @@ defmodule Explorer.ChainTest do |> insert(to_address: address) |> with_block() - insert(:log, block: transaction1.block, transaction: transaction1, index: 1, address: address) + insert(:log, + block: transaction1.block, + transaction: transaction1, + index: 1, + address: address, + block_number: transaction1.block_number + ) transaction2 = :transaction |> insert(from_address: address) |> with_block() - insert(:log, block: transaction2.block, transaction: transaction2, index: 2, address: address, first_topic: "test") + insert(:log, + block: transaction2.block, + transaction: transaction2, + index: 2, + address: address, + first_topic: "test", + block_number: transaction2.block_number + ) [found_log] = Chain.address_to_logs(address_hash, topic: "test") @@ -305,6 +336,7 @@ defmodule Explorer.ChainTest do insert(:log, block: transaction1.block, + block_number: transaction1.block_number, transaction: transaction1, index: 1, address: address, @@ -316,7 +348,13 @@ defmodule Explorer.ChainTest do |> insert(from_address: address) |> with_block() - insert(:log, block: transaction2.block, transaction: transaction2, index: 2, address: address) + insert(:log, + block: transaction2.block, + block_number: transaction2.block.number, + transaction: transaction2, + index: 2, + address: address + ) [found_log] = Chain.address_to_logs(address_hash, topic: "test") @@ -2537,7 +2575,8 @@ defmodule Explorer.ChainTest do |> insert() |> with_block() - %Log{transaction_hash: transaction_hash, index: index} = insert(:log, transaction: transaction) + %Log{transaction_hash: transaction_hash, index: index} = + insert(:log, transaction: transaction, block: transaction.block, block_number: transaction.block_number) assert [%Log{transaction_hash: ^transaction_hash, index: ^index}] = Chain.transaction_to_logs(transaction.hash) end @@ -2548,11 +2587,24 @@ defmodule Explorer.ChainTest do |> insert() |> with_block() - log = insert(:log, transaction: transaction, index: 1) + log = + insert(:log, + transaction: transaction, + index: 1, + block: transaction.block, + block_number: transaction.block_number + ) second_page_indexes = 2..51 - |> Enum.map(fn index -> insert(:log, transaction: transaction, index: index) end) + |> Enum.map(fn index -> + insert(:log, + transaction: transaction, + index: index, + block: transaction.block, + block_number: transaction.block_number + ) + end) |> Enum.map(& &1.index) assert second_page_indexes == @@ -2567,7 +2619,7 @@ defmodule Explorer.ChainTest do |> insert() |> with_block() - insert(:log, transaction: transaction) + insert(:log, transaction: transaction, block: transaction.block, block_number: transaction.block_number) assert [%Log{address: %Address{}, transaction: %Transaction{}}] = Chain.transaction_to_logs( From 3f1d6a8d1ced4fc98bbc6f0656073c71803d353e Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Wed, 27 Nov 2019 09:16:49 +0300 Subject: [PATCH 43/57] fix viewing_transactions_test --- .../test/block_scout_web/features/viewing_transactions_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/block_scout_web/test/block_scout_web/features/viewing_transactions_test.exs b/apps/block_scout_web/test/block_scout_web/features/viewing_transactions_test.exs index 26b7e07a73..6775065f17 100644 --- a/apps/block_scout_web/test/block_scout_web/features/viewing_transactions_test.exs +++ b/apps/block_scout_web/test/block_scout_web/features/viewing_transactions_test.exs @@ -44,7 +44,7 @@ defmodule BlockScoutWeb.ViewingTransactionsTest do ) |> with_block(block, gas_used: Decimal.new(1_230_000_000_000_123_000), status: :ok) - insert(:log, address: lincoln, index: 0, transaction: transaction) + insert(:log, address: lincoln, index: 0, transaction: transaction, block: block, block_number: block.number) internal = insert(:internal_transaction, From 6ea228eec4e254fb70548ccff5cc2d09312bf015 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Wed, 27 Nov 2019 09:52:27 +0300 Subject: [PATCH 44/57] add index on block_number --- .../20191121064805_add_block_hash_and_block_index_to_logs.exs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/explorer/priv/repo/migrations/20191121064805_add_block_hash_and_block_index_to_logs.exs b/apps/explorer/priv/repo/migrations/20191121064805_add_block_hash_and_block_index_to_logs.exs index 3856339771..ce74a97311 100644 --- a/apps/explorer/priv/repo/migrations/20191121064805_add_block_hash_and_block_index_to_logs.exs +++ b/apps/explorer/priv/repo/migrations/20191121064805_add_block_hash_and_block_index_to_logs.exs @@ -7,6 +7,8 @@ defmodule Explorer.Repo.Migrations.AddBlockHashAndBlockIndexToLogs do add(:block_number, :integer) end + create(index(:logs, [:block_number])) + execute(""" UPDATE logs log SET block_hash = with_block.block_hash, From 3dd80bb9b9e9c615a79928f092090c5d717602a2 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Wed, 27 Nov 2019 11:31:22 +0300 Subject: [PATCH 45/57] drop unique index on transaction_hash, index --- .../20191121064805_add_block_hash_and_block_index_to_logs.exs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/explorer/priv/repo/migrations/20191121064805_add_block_hash_and_block_index_to_logs.exs b/apps/explorer/priv/repo/migrations/20191121064805_add_block_hash_and_block_index_to_logs.exs index ce74a97311..4771653646 100644 --- a/apps/explorer/priv/repo/migrations/20191121064805_add_block_hash_and_block_index_to_logs.exs +++ b/apps/explorer/priv/repo/migrations/20191121064805_add_block_hash_and_block_index_to_logs.exs @@ -34,5 +34,9 @@ defmodule Explorer.Repo.Migrations.AddBlockHashAndBlockIndexToLogs do DROP CONSTRAINT logs_pkey, ADD PRIMARY KEY (transaction_hash, block_hash, index); """) + + drop(unique_index(:logs, [:transaction_hash, :index])) + + create_if_not_exists(index(:logs, [:transaction_hash, :index])) end end From 3ed0e82d245f0b4b05eb0a0da25cc58d179810ac Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Thu, 28 Nov 2019 10:26:25 +0300 Subject: [PATCH 46/57] change migration order --- .../20191121064805_add_block_hash_and_block_index_to_logs.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/explorer/priv/repo/migrations/20191121064805_add_block_hash_and_block_index_to_logs.exs b/apps/explorer/priv/repo/migrations/20191121064805_add_block_hash_and_block_index_to_logs.exs index 4771653646..bd6d392e5a 100644 --- a/apps/explorer/priv/repo/migrations/20191121064805_add_block_hash_and_block_index_to_logs.exs +++ b/apps/explorer/priv/repo/migrations/20191121064805_add_block_hash_and_block_index_to_logs.exs @@ -7,8 +7,6 @@ defmodule Explorer.Repo.Migrations.AddBlockHashAndBlockIndexToLogs do add(:block_number, :integer) end - create(index(:logs, [:block_number])) - execute(""" UPDATE logs log SET block_hash = with_block.block_hash, @@ -25,6 +23,8 @@ defmodule Explorer.Repo.Migrations.AddBlockHashAndBlockIndexToLogs do ; """) + create(index(:logs, [:block_number])) + alter table(:logs) do modify(:block_hash, references(:blocks, column: :hash, type: :bytea), null: false) end From 345063d2ad7b3fbb9547034b5e880999ccbb2f26 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Thu, 28 Nov 2019 17:28:44 +0300 Subject: [PATCH 47/57] fix merging conflicts --- apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/log.ex | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/log.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/log.ex index 11f91b1fec..6c4345dbff 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/log.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/log.ex @@ -100,7 +100,6 @@ defmodule EthereumJSONRPC.Log do block_number: block_number, block_hash: block_hash, data: data, - block_hash: block_hash, index: index, transaction_hash: transaction_hash } From ca0d57f40d0fb257de8031c7b5e323f359d0ac16 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Fri, 29 Nov 2019 16:17:07 +0300 Subject: [PATCH 48/57] remove invalid logs in migration --- .../20191121064805_add_block_hash_and_block_index_to_logs.exs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/explorer/priv/repo/migrations/20191121064805_add_block_hash_and_block_index_to_logs.exs b/apps/explorer/priv/repo/migrations/20191121064805_add_block_hash_and_block_index_to_logs.exs index bd6d392e5a..e873fb5951 100644 --- a/apps/explorer/priv/repo/migrations/20191121064805_add_block_hash_and_block_index_to_logs.exs +++ b/apps/explorer/priv/repo/migrations/20191121064805_add_block_hash_and_block_index_to_logs.exs @@ -25,6 +25,10 @@ defmodule Explorer.Repo.Migrations.AddBlockHashAndBlockIndexToLogs do create(index(:logs, [:block_number])) + execute(""" + DELETE FROM logs WHERE block_hash IS NULL; + """) + alter table(:logs) do modify(:block_hash, references(:blocks, column: :hash, type: :bytea), null: false) end From 49e4a803f352dfb3b26ab064768d0036ad881695 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Mon, 9 Dec 2019 17:27:57 +0300 Subject: [PATCH 49/57] remove non-consensus-records from pending ops periodically --- .../lib/explorer/chain/import/runner/blocks.ex | 3 --- .../lib/indexer/fetcher/internal_transaction.ex | 16 ++++++++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex index 7d7c7e0ca1..5253256f41 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex @@ -8,7 +8,6 @@ defmodule Explorer.Chain.Import.Runner.Blocks do import Ecto.Query, only: [from: 2, subquery: 1] alias Ecto.{Changeset, Multi, Repo} - alias Explorer.Chain alias Explorer.Chain.{Address, Block, Import, PendingBlockOperation, Transaction} alias Explorer.Chain.Block.Reward alias Explorer.Chain.Import.Runner @@ -303,8 +302,6 @@ defmodule Explorer.Chain.Import.Runner.Blocks do timeout: timeout ) - :ok = Chain.remove_nonconsensus_blocks_from_pending_ops(removed_consensus_block_hashes) - {:ok, removed_consensus_block_hashes} rescue postgrex_error in Postgrex.Error -> diff --git a/apps/indexer/lib/indexer/fetcher/internal_transaction.ex b/apps/indexer/lib/indexer/fetcher/internal_transaction.ex index aa9a967cd1..5107a3b758 100644 --- a/apps/indexer/lib/indexer/fetcher/internal_transaction.ex +++ b/apps/indexer/lib/indexer/fetcher/internal_transaction.ex @@ -76,9 +76,25 @@ defmodule Indexer.Fetcher.InternalTransaction do reducer.(block_number, acc) end) + schedule_next_removal_nonconsensus_pending_ops() + final end + defp schedule_next_removal_nonconsensus_pending_ops do + Process.send_after(self(), :remove_nonconsensus_blocks_from_pending_transactions, :timer.minutes(30)) + end + + def handle_info(:remove_nonconsensus_blocks_from_pending_transactions, state) do + Task.async(fn -> + :ok = Chain.remove_nonconsensus_blocks_from_pending_ops() + end) + + schedule_next_removal_nonconsensus_pending_ops() + + {:noreply, state} + end + defp params(%{block_number: block_number, hash: hash, index: index}) when is_integer(block_number) do %{block_number: block_number, hash_data: to_string(hash), transaction_index: index} end From 767d8c999930287df651b5b66f457588f4542f3a Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 10 Dec 2019 15:44:41 +0300 Subject: [PATCH 50/57] add separate process for pending ops removal --- .../indexer/fetcher/internal_transaction.ex | 16 ------- .../lib/indexer/pending_ops_cleaner.ex | 45 +++++++++++++++++++ .../test/indexer/pending_ops_cleaner_test.exs | 34 ++++++++++++++ 3 files changed, 79 insertions(+), 16 deletions(-) create mode 100644 apps/indexer/lib/indexer/pending_ops_cleaner.ex create mode 100644 apps/indexer/test/indexer/pending_ops_cleaner_test.exs diff --git a/apps/indexer/lib/indexer/fetcher/internal_transaction.ex b/apps/indexer/lib/indexer/fetcher/internal_transaction.ex index 5107a3b758..aa9a967cd1 100644 --- a/apps/indexer/lib/indexer/fetcher/internal_transaction.ex +++ b/apps/indexer/lib/indexer/fetcher/internal_transaction.ex @@ -76,25 +76,9 @@ defmodule Indexer.Fetcher.InternalTransaction do reducer.(block_number, acc) end) - schedule_next_removal_nonconsensus_pending_ops() - final end - defp schedule_next_removal_nonconsensus_pending_ops do - Process.send_after(self(), :remove_nonconsensus_blocks_from_pending_transactions, :timer.minutes(30)) - end - - def handle_info(:remove_nonconsensus_blocks_from_pending_transactions, state) do - Task.async(fn -> - :ok = Chain.remove_nonconsensus_blocks_from_pending_ops() - end) - - schedule_next_removal_nonconsensus_pending_ops() - - {:noreply, state} - end - defp params(%{block_number: block_number, hash: hash, index: index}) when is_integer(block_number) do %{block_number: block_number, hash_data: to_string(hash), transaction_index: index} end diff --git a/apps/indexer/lib/indexer/pending_ops_cleaner.ex b/apps/indexer/lib/indexer/pending_ops_cleaner.ex new file mode 100644 index 0000000000..dd3e39950c --- /dev/null +++ b/apps/indexer/lib/indexer/pending_ops_cleaner.ex @@ -0,0 +1,45 @@ +defmodule Indexer.PendingOpsCleaner do + @moduledoc """ + Peiodically cleans non-consensus pending ops. + """ + + use GenServer + + require Logger + + alias Explorer.Chain + + @interval :timer.minutes(60) + + def start_link([init_opts, gen_server_opts]) do + start_link(init_opts, gen_server_opts) + end + + def start_link(init_opts, gen_server_opts) do + GenServer.start_link(__MODULE__, init_opts, gen_server_opts) + end + + def init(opts) do + interval = opts[:interval] || @interval + + Process.send_after(self(), :clean_nonconsensus_pending_ops, interval) + + {:ok, %{interval: interval}} + end + + def handle_info(:clean_nonconsensus_pending_ops, %{interval: interval} = state) do + Logger.debug(fn -> "Cleaning non-consensus pending ops" end) + + clean_nonconsensus_pending_ops() + + Process.send_after(self(), :clean_nonconsensus_pending_ops, interval) + + {:noreply, state} + end + + defp clean_nonconsensus_pending_ops do + :ok = Chain.remove_nonconsensus_blocks_from_pending_ops() + + Logger.debug(fn -> "Non-consensus pending ops are cleaned" end) + end +end diff --git a/apps/indexer/test/indexer/pending_ops_cleaner_test.exs b/apps/indexer/test/indexer/pending_ops_cleaner_test.exs new file mode 100644 index 0000000000..f8f1cbdb30 --- /dev/null +++ b/apps/indexer/test/indexer/pending_ops_cleaner_test.exs @@ -0,0 +1,34 @@ +defmodule Indexer.PendingOpsCleanerTest do + use Explorer.DataCase + + alias Explorer.Chain.PendingBlockOperation + alias Indexer.PendingOpsCleaner + + describe "init/1" do + test "deletes non-consensus pending ops on init" do + block = insert(:block, consensus: false) + + insert(:pending_block_operation, block_hash: block.hash, fetch_internal_transactions: true) + + assert Repo.one(from(block in PendingBlockOperation, where: block.block_hash == ^block.hash)) + + start_supervised!({PendingOpsCleaner, [[interval: :timer.seconds(1)], [name: :PendingOpsTest]]}) + + Process.sleep(2_000) + + assert is_nil(Repo.one(from(block in PendingBlockOperation, where: block.block_hash == ^block.hash))) + end + + test "re-schedules deletion" do + start_supervised!({PendingOpsCleaner, [[interval: :timer.seconds(1)], [name: :PendingOpsTest]]}) + + block = insert(:block, consensus: false) + + insert(:pending_block_operation, block_hash: block.hash, fetch_internal_transactions: true) + + Process.sleep(2_000) + + assert is_nil(Repo.one(from(block in PendingBlockOperation, where: block.block_hash == ^block.hash))) + end + end +end From 40ad32eb7248ed5dc0482efa3575f1bdc38ab62d Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 10 Dec 2019 16:02:18 +0300 Subject: [PATCH 51/57] add PendingOpsCleaner to Supervision tree --- apps/indexer/lib/indexer/supervisor.ex | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/indexer/lib/indexer/supervisor.ex b/apps/indexer/lib/indexer/supervisor.ex index f922a1a180..f22b6de487 100644 --- a/apps/indexer/lib/indexer/supervisor.ex +++ b/apps/indexer/lib/indexer/supervisor.ex @@ -5,7 +5,7 @@ defmodule Indexer.Supervisor do use Supervisor - alias Indexer.Block + alias Indexer.{Block, PendingOpsCleaner} alias Indexer.Block.{Catchup, Realtime} alias Indexer.Fetcher.{ @@ -129,7 +129,8 @@ defmodule Indexer.Supervisor do {UnclesWithoutIndex.Supervisor, [[json_rpc_named_arguments: json_rpc_named_arguments, memory_monitor: memory_monitor]]}, {BlocksTransactionsMismatch.Supervisor, - [[json_rpc_named_arguments: json_rpc_named_arguments, memory_monitor: memory_monitor]]} + [[json_rpc_named_arguments: json_rpc_named_arguments, memory_monitor: memory_monitor]]}, + {PendingOpsCleaner, [[], []]} ], strategy: :one_for_one ) From 12905cf9a521bf00992399fde585a21afaa52bca Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Wed, 18 Dec 2019 18:05:44 +0300 Subject: [PATCH 52/57] Speedup address_to_logs query --- CHANGELOG.md | 3 +-- apps/explorer/lib/explorer/chain.ex | 23 +++++++++++++------ ...18120138_logs_block_number_index_index.exs | 9 ++++++++ 3 files changed, 26 insertions(+), 9 deletions(-) create mode 100644 apps/explorer/priv/repo/migrations/20191218120138_logs_block_number_index_index.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index ec51506d9c..8675c5c51e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,12 +3,11 @@ ### Features ### Fixes +- [#2924](https://github.com/poanetwork/blockscout/pull/2924) - Speedup address to logs query - [#2906](https://github.com/poanetwork/blockscout/pull/2906) - fix address sum cache - - [#2902](https://github.com/poanetwork/blockscout/pull/2902) - Offset in blocks retrieval for average block time ### Chore - - [#2896](https://github.com/poanetwork/blockscout/pull/2896) - Disable Parity websockets tests diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index ca4bc2a1f5..2ca7d17925 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -358,22 +358,31 @@ defmodule Explorer.Chain do base_query = from(log in Log, inner_join: transaction in Transaction, - on: - transaction.block_hash == log.block_hash and transaction.block_number == log.block_number and - transaction.hash == log.transaction_hash, - order_by: [desc: transaction.block_number, desc: transaction.index], - preload: [:transaction, transaction: [to_address: :smart_contract]], + on: transaction.hash == log.transaction_hash, + order_by: [desc: log.block_number, desc: log.index], where: transaction.block_number < ^block_number, or_where: transaction.block_number == ^block_number and transaction.index > ^transaction_index, or_where: transaction.block_number == ^block_number and transaction.index == ^transaction_index and log.index > ^log_index, - where: log.address_hash == ^address_hash and log.block_hash == transaction.block_hash, + where: log.address_hash == ^address_hash, limit: ^paging_options.page_size, select: log ) - base_query + wrapped_query = + from( + log in subquery(base_query), + inner_join: transaction in Transaction, + preload: [:transaction, transaction: [to_address: :smart_contract]], + where: + log.block_hash == transaction.block_hash and + log.block_number == transaction.block_number and + log.transaction_hash == transaction.hash, + select: log + ) + + wrapped_query |> filter_topic(options) |> Repo.all() |> Enum.take(paging_options.page_size) diff --git a/apps/explorer/priv/repo/migrations/20191218120138_logs_block_number_index_index.exs b/apps/explorer/priv/repo/migrations/20191218120138_logs_block_number_index_index.exs new file mode 100644 index 0000000000..14d0b647a0 --- /dev/null +++ b/apps/explorer/priv/repo/migrations/20191218120138_logs_block_number_index_index.exs @@ -0,0 +1,9 @@ +defmodule Explorer.Repo.Migrations.LogsBlockNumberIndexIndex do + use Ecto.Migration + + def change do + create_if_not_exists(index(:logs, ["block_number DESC, index DESC"])) + + drop_if_exists(index(:logs, [:block_number], name: "logs_block_number_index")) + end +end From bccf71c7484de35e42c2cf17eaaa51712ea46873 Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Wed, 18 Dec 2019 19:07:31 +0300 Subject: [PATCH 53/57] New indexing scheme update migration --- CHANGELOG.md | 6 +-- ...054_add_pending_internal_txs_operation.exs | 52 ++++++++++++++++++- ...add_block_hash_and_block_index_to_logs.exs | 5 +- ...2035_add_block_hash_to_token_transfers.exs | 7 ++- 4 files changed, 57 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ec51506d9c..c7f9f655fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,14 +1,13 @@ ## Current ### Features +- [#2835](https://github.com/poanetwork/blockscout/pull/2835), [#2871](https://github.com/poanetwork/blockscout/pull/2871), [#2872](https://github.com/poanetwork/blockscout/pull/2872), [#2886](https://github.com/poanetwork/blockscout/pull/2886), [#2925](https://github.com/poanetwork/blockscout/pull/2925) - Add "block_hash" to logs, token_transfers and internal transactions and "pending blocks operations" approach ### Fixes - [#2906](https://github.com/poanetwork/blockscout/pull/2906) - fix address sum cache - - [#2902](https://github.com/poanetwork/blockscout/pull/2902) - Offset in blocks retrieval for average block time ### Chore - - [#2896](https://github.com/poanetwork/blockscout/pull/2896) - Disable Parity websockets tests @@ -34,8 +33,6 @@ - [#2830](https://github.com/poanetwork/blockscout/pull/2830) - Fix wrong color of contract icon on xDai chain - [#2829](https://github.com/poanetwork/blockscout/pull/2829) - Fix for stuck gas limit label and value - [#2828](https://github.com/poanetwork/blockscout/pull/2828) - Fix for script that clears compilation/launching assets -- [#2872](https://github.com/poanetwork/blockscout/pull/2872) - do not remove token transfers -- [#2871](https://github.com/poanetwork/blockscout/pull/2871) - do not remove logs - [#2800](https://github.com/poanetwork/blockscout/pull/2800) - return not found for not verified contract for token read_contract - [#2806](https://github.com/poanetwork/blockscout/pull/2806) - Fix blocks fetching on the main page - [#2803](https://github.com/poanetwork/blockscout/pull/2803) - Fix block validator custom tooltip @@ -51,7 +48,6 @@ - [#2844](https://github.com/poanetwork/blockscout/pull/2844) - Extend external reward types up to 20 - [#2827](https://github.com/poanetwork/blockscout/pull/2827) - Node js 12.13.0 (latest LTS release) support - [#2818](https://github.com/poanetwork/blockscout/pull/2818) - allow hiding marketcap percentage -- [#2835](https://github.com/poanetwork/blockscout/pull/2835) - Update int txs constraints repairing script - [#2817](https://github.com/poanetwork/blockscout/pull/2817) - move docker integration documentation to blockscout docs - [#2808](https://github.com/poanetwork/blockscout/pull/2808) - Add tooltip for tx input - [#2807](https://github.com/poanetwork/blockscout/pull/2807) - 422 page diff --git a/apps/explorer/priv/repo/migrations/20191018140054_add_pending_internal_txs_operation.exs b/apps/explorer/priv/repo/migrations/20191018140054_add_pending_internal_txs_operation.exs index a12fa48836..d603d1e853 100644 --- a/apps/explorer/priv/repo/migrations/20191018140054_add_pending_internal_txs_operation.exs +++ b/apps/explorer/priv/repo/migrations/20191018140054_add_pending_internal_txs_operation.exs @@ -48,8 +48,54 @@ defmodule Explorer.Repo.Migrations.AddPendingInternalTxsOperation do ON t.hash = i.transaction_hash ) AS with_block WHERE itx.transaction_hash = with_block.transaction_hash - AND itx.index = with_block.index - ; + AND itx.index = with_block.index; + """) + + execute(""" + DELETE FROM internal_transactions WHERE block_hash IS NULL; + """) + + execute(""" + DO $$ + DECLARE + duplicates_count INTEGER := 0; + blocks_scanned INTEGER := 0; + temprow RECORD; + BEGIN + FOR temprow IN + SELECT number, hash FROM blocks + LOOP + blocks_scanned := blocks_scanned + 1; + IF EXISTS ( + SELECT 1 FROM transactions WHERE block_hash = temprow.hash + ) THEN + IF EXISTS ( + SELECT block_hash, block_index FROM internal_transactions + WHERE block_hash = temprow.hash + GROUP BY block_hash, block_index HAVING COUNT(*) > 1 + ) THEN + duplicates_count := duplicates_count + 1; + RAISE NOTICE '% duplicates, blocks scanned %, block #%, block hash is %', duplicates_count, blocks_scanned, temprow.number , temprow.hash; + + IF NOT EXISTS ( + SELECT 1 FROM pending_block_operations + WHERE block_hash = temprow.hash + ) THEN + INSERT INTO pending_block_operations + (block_hash, inserted_at, updated_at, fetch_internal_transactions) + SELECT b.hash, now(), now(), TRUE FROM blocks b + WHERE b.hash = temprow.hash; + END IF; + + DELETE FROM internal_transactions + WHERE block_hash = temprow.hash; + + RAISE NOTICE 'DELETED'; + END IF; + END IF; + END LOOP; + RAISE NOTICE 'SCRIPT FINISHED'; + END $$; """) execute(""" @@ -70,5 +116,7 @@ defmodule Explorer.Repo.Migrations.AddPendingInternalTxsOperation do name: :internal_transactions_transaction_hash_index_index ) ) + + create_if_not_exists(index(:internal_transactions, [:transaction_hash, :index])) end end diff --git a/apps/explorer/priv/repo/migrations/20191121064805_add_block_hash_and_block_index_to_logs.exs b/apps/explorer/priv/repo/migrations/20191121064805_add_block_hash_and_block_index_to_logs.exs index e873fb5951..94033d52d0 100644 --- a/apps/explorer/priv/repo/migrations/20191121064805_add_block_hash_and_block_index_to_logs.exs +++ b/apps/explorer/priv/repo/migrations/20191121064805_add_block_hash_and_block_index_to_logs.exs @@ -19,12 +19,9 @@ defmodule Explorer.Repo.Migrations.AddBlockHashAndBlockIndexToLogs do JOIN transactions t ON t.hash = l.transaction_hash ) AS with_block - WHERE log.transaction_hash = with_block.transaction_hash - ; + WHERE log.transaction_hash = with_block.transaction_hash; """) - create(index(:logs, [:block_number])) - execute(""" DELETE FROM logs WHERE block_hash IS NULL; """) diff --git a/apps/explorer/priv/repo/migrations/20191122062035_add_block_hash_to_token_transfers.exs b/apps/explorer/priv/repo/migrations/20191122062035_add_block_hash_to_token_transfers.exs index 5635cd9bd9..c10d3b2713 100644 --- a/apps/explorer/priv/repo/migrations/20191122062035_add_block_hash_to_token_transfers.exs +++ b/apps/explorer/priv/repo/migrations/20191122062035_add_block_hash_to_token_transfers.exs @@ -16,8 +16,11 @@ defmodule Explorer.Repo.Migrations.AddBlockHashToTokenTransfers do JOIN transactions t ON t.hash = transfer.transaction_hash ) AS with_block - WHERE token_transfer.transaction_hash = with_block.transaction_hash - ; + WHERE token_transfer.transaction_hash = with_block.transaction_hash; + """) + + execute(""" + DELETE FROM token_transfers WHERE block_hash IS NULL; """) alter table(:token_transfers) do From 55d280c05ca535c61dc1df59db2c8d4ef4533dd8 Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Fri, 20 Dec 2019 14:34:01 +0300 Subject: [PATCH 54/57] Partial index on pending_block_operations table on block hash where fetch_internal_transactions --- CHANGELOG.md | 3 +-- ...pending_block_operations_block_hash_partial_index.exs | 9 +++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 apps/explorer/priv/repo/migrations/20191220113006_pending_block_operations_block_hash_partial_index.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index ec51506d9c..811ca59583 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,12 +3,11 @@ ### Features ### Fixes +- [#2928](https://github.com/poanetwork/blockscout/pull/2928) - Speedup pending block ops int txs to fetch query - [#2906](https://github.com/poanetwork/blockscout/pull/2906) - fix address sum cache - - [#2902](https://github.com/poanetwork/blockscout/pull/2902) - Offset in blocks retrieval for average block time ### Chore - - [#2896](https://github.com/poanetwork/blockscout/pull/2896) - Disable Parity websockets tests diff --git a/apps/explorer/priv/repo/migrations/20191220113006_pending_block_operations_block_hash_partial_index.exs b/apps/explorer/priv/repo/migrations/20191220113006_pending_block_operations_block_hash_partial_index.exs new file mode 100644 index 0000000000..1b1a52e4ed --- /dev/null +++ b/apps/explorer/priv/repo/migrations/20191220113006_pending_block_operations_block_hash_partial_index.exs @@ -0,0 +1,9 @@ +defmodule Explorer.Repo.Migrations.PendingBlockOperationsBlockHashPartialIndex do + use Ecto.Migration + + def change do + execute( + "CREATE INDEX pending_block_operations_block_hash_index_partial ON pending_block_operations(block_hash) WHERE fetch_internal_transactions=true;" + ) + end +end From 06049209f111c2fc1395e1e5c805ea84ca385a99 Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Fri, 20 Dec 2019 16:09:27 +0300 Subject: [PATCH 55/57] Remove dropping of index logs_block_number_index because it never exists in migrations now --- .../migrations/20191218120138_logs_block_number_index_index.exs | 2 -- 1 file changed, 2 deletions(-) diff --git a/apps/explorer/priv/repo/migrations/20191218120138_logs_block_number_index_index.exs b/apps/explorer/priv/repo/migrations/20191218120138_logs_block_number_index_index.exs index 14d0b647a0..735a835db9 100644 --- a/apps/explorer/priv/repo/migrations/20191218120138_logs_block_number_index_index.exs +++ b/apps/explorer/priv/repo/migrations/20191218120138_logs_block_number_index_index.exs @@ -3,7 +3,5 @@ defmodule Explorer.Repo.Migrations.LogsBlockNumberIndexIndex do def change do create_if_not_exists(index(:logs, ["block_number DESC, index DESC"])) - - drop_if_exists(index(:logs, [:block_number], name: "logs_block_number_index")) end end From cd5011aa4d66d127a905cce674ac61f5ad56d12a Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Tue, 31 Dec 2019 15:02:39 +0300 Subject: [PATCH 56/57] Fix getsourcecode test --- .../controllers/api/rpc/contract_controller_test.exs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/contract_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/contract_controller_test.exs index 5c12c1e5db..505520a2c0 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/contract_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/contract_controller_test.exs @@ -524,6 +524,8 @@ defmodule BlockScoutWeb.API.RPC.ContractControllerTest do created_contract_address: created_contract_address, created_contract_code: smart_contract_bytecode, block_number: transaction.block_number, + block_hash: transaction.block_hash, + block_index: 0, transaction_index: transaction.index ) From 2aa00386044ab7db7d50306f25915c0cb003a416 Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Tue, 31 Dec 2019 15:11:08 +0300 Subject: [PATCH 57/57] Update CHANGELOG --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8950cf4619..20baebf3be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,13 +2,13 @@ ### Features - [#2835](https://github.com/poanetwork/blockscout/pull/2835), [#2871](https://github.com/poanetwork/blockscout/pull/2871), [#2872](https://github.com/poanetwork/blockscout/pull/2872), [#2886](https://github.com/poanetwork/blockscout/pull/2886), [#2925](https://github.com/poanetwork/blockscout/pull/2925) - Add "block_hash" to logs, token_transfers and internal transactions and "pending blocks operations" approach +- [#2926](https://github.com/poanetwork/blockscout/pull/2926) - API endpoint: sum balances except burnt address +- [#2918](https://github.com/poanetwork/blockscout/pull/2918) - Add tokenID for tokentx API action explicitly ### Fixes - [#2932](https://github.com/poanetwork/blockscout/pull/2932) - fix duplicate websocket connection - [#2928](https://github.com/poanetwork/blockscout/pull/2928) - Speedup pending block ops int txs to fetch query -- [#2926](https://github.com/poanetwork/blockscout/pull/2926) - API endpoint: sum balances except burnt address - [#2924](https://github.com/poanetwork/blockscout/pull/2924) - Speedup address to logs query -- [#2918](https://github.com/poanetwork/blockscout/pull/2918) - Add tokenID for tokentx API action - [#2915](https://github.com/poanetwork/blockscout/pull/2915) - Speedup of blocks_without_reward_query - [#2914](https://github.com/poanetwork/blockscout/pull/2914) - Reduce execution time of stream_unfetched_token_instances query - [#2906](https://github.com/poanetwork/blockscout/pull/2906) - fix address sum cache