From 70147b7155fc6bea130bb0bd101111948fb09c87 Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Mon, 2 Dec 2019 20:25:58 +0300 Subject: [PATCH 1/8] Increase performance of rewards query --- CHANGELOG.md | 1 + apps/explorer/lib/explorer/chain/block/reward.ex | 10 ++++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 62ce88bd8d..1c4637e6eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ### Features ### Fixes +- [#2908](https://github.com/poanetwork/blockscout/pull/2908) - Fix performance of address page ### Chore diff --git a/apps/explorer/lib/explorer/chain/block/reward.ex b/apps/explorer/lib/explorer/chain/block/reward.ex index bc38f1ad08..df94253676 100644 --- a/apps/explorer/lib/explorer/chain/block/reward.ex +++ b/apps/explorer/lib/explorer/chain/block/reward.ex @@ -112,9 +112,15 @@ defmodule Explorer.Chain.Block.Reward do end defp join_associations(query) do + last_blocks_query = + from(block in Block, + limit: 1000, + order_by: [desc: block.number] + ) + query |> preload(:address) - |> join(:inner, [reward], block in assoc(reward, :block)) - |> preload(:block) + |> join(:inner, [reward], block in subquery(last_blocks_query), on: block.hash == reward.block_hash) + |> preload(block: ^last_blocks_query) end end From f8f409b2a41e9810b292c0213dac7110923ba04f Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Wed, 4 Dec 2019 17:19:34 +0300 Subject: [PATCH 2/8] Increase to 2k limit for rewards --- apps/explorer/lib/explorer/chain/block/reward.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/explorer/lib/explorer/chain/block/reward.ex b/apps/explorer/lib/explorer/chain/block/reward.ex index df94253676..aa73ab4cb1 100644 --- a/apps/explorer/lib/explorer/chain/block/reward.ex +++ b/apps/explorer/lib/explorer/chain/block/reward.ex @@ -114,7 +114,7 @@ defmodule Explorer.Chain.Block.Reward do defp join_associations(query) do last_blocks_query = from(block in Block, - limit: 1000, + limit: 2000, order_by: [desc: block.number] ) From d3e258c43173621837802d4a8cc8acb93835b590 Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Tue, 3 Dec 2019 14:35:36 +0300 Subject: [PATCH 3/8] Reorganize queries and indexes for internal_transactions table --- CHANGELOG.md | 2 + .../api/rpc/address_controller_test.exs | 8 +- ...n_internal_transaction_controller_test.exs | 3 +- apps/explorer/lib/explorer/chain.ex | 66 +++++++++++- .../explorer/chain/internal_transaction.ex | 12 +++ apps/explorer/lib/explorer/etherscan.ex | 102 ++++++++++++++---- ...transactions_add_to_address_hash_index.exs | 25 +++++ apps/explorer/test/explorer/chain_test.exs | 26 +++-- .../explorer/test/explorer/etherscan_test.exs | 16 +-- 9 files changed, 218 insertions(+), 42 deletions(-) create mode 100644 apps/explorer/priv/repo/migrations/20191203112646_internal_transactions_add_to_address_hash_index.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index 62ce88bd8d..ad65137203 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ ### Fixes +- [2910](https://github.com/poanetwork/blockscout/pull/2910) - Reorganize queries and indexes for internal_transactions table + ### Chore - [#2896](https://github.com/poanetwork/blockscout/pull/2896) - Disable Parity websockets tests diff --git a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/address_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/address_controller_test.exs index 135fbe9948..66b98cc661 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/address_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/address_controller_test.exs @@ -1606,7 +1606,7 @@ defmodule BlockScoutWeb.API.RPC.AddressControllerTest do internal_transaction = :internal_transaction_create - |> insert(transaction: transaction, index: 0, from_address: address) + |> insert(transaction: transaction, index: 0, from_address: address, block_number: block.number) |> with_contract_creation(contract_address) params = %{ @@ -1658,7 +1658,8 @@ defmodule BlockScoutWeb.API.RPC.AddressControllerTest do transaction: transaction, index: 0, type: :reward, - error: "some error" + error: "some error", + block_number: transaction.block_number ] insert(:internal_transaction_create, internal_transaction_details) @@ -1693,7 +1694,8 @@ defmodule BlockScoutWeb.API.RPC.AddressControllerTest do internal_transaction_details = %{ from_address: address, transaction: transaction, - index: index + index: index, + block_number: transaction.block_number } insert(:internal_transaction_create, internal_transaction_details) diff --git a/apps/block_scout_web/test/block_scout_web/controllers/transaction_internal_transaction_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/transaction_internal_transaction_controller_test.exs index 773b51a36c..9d6093e468 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/transaction_internal_transaction_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/transaction_internal_transaction_controller_test.exs @@ -64,7 +64,8 @@ defmodule BlockScoutWeb.TransactionInternalTransactionControllerTest do assert json_response(conn, 200) - assert Enum.count(items) == 2 + # excluding of internal transactions with type=call and index=0 + assert Enum.count(items) == 1 end test "includes USD exchange rate value for address in assigns", %{conn: conn} do diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index 9576eca1a6..e7fdf09330 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -15,6 +15,7 @@ defmodule Explorer.Chain do preload: 2, select: 2, subquery: 1, + union: 2, union_all: 2, where: 2, where: 3 @@ -193,8 +194,65 @@ defmodule Explorer.Chain do direction = Keyword.get(options, :direction) paging_options = Keyword.get(options, :paging_options, @default_paging_options) - InternalTransaction - |> InternalTransaction.where_address_fields_match(hash, direction) + if direction == nil do + query_to_address_hash_wrapped = + InternalTransaction + |> InternalTransaction.where_address_fields_match(hash, :to_address_hash) + |> common_where_limit_order(paging_options) + |> wrapped_union_subquery() + + query_from_address_hash_wrapped = + InternalTransaction + |> InternalTransaction.where_address_fields_match(hash, :from_address_hash) + |> common_where_limit_order(paging_options) + |> wrapped_union_subquery() + + query_created_contract_address_hash_wrapped = + InternalTransaction + |> InternalTransaction.where_address_fields_match(hash, :created_contract_address_hash) + |> common_where_limit_order(paging_options) + |> wrapped_union_subquery() + + full_query = + query_to_address_hash_wrapped + |> union(^query_from_address_hash_wrapped) + |> union(^query_created_contract_address_hash_wrapped) + + full_wrapped_query = + from( + q in subquery(full_query), + select: q + ) + + full_wrapped_query + |> order_by( + [q], + desc: q.block_number, + desc: q.transaction_index, + desc: q.index + ) + |> preload(transaction: :block) + |> join_associations(necessity_by_association) + |> Repo.all() + else + InternalTransaction + |> InternalTransaction.where_address_fields_match(hash, direction) + |> common_where_limit_order(paging_options) + |> preload(transaction: :block) + |> join_associations(necessity_by_association) + |> Repo.all() + end + end + + def wrapped_union_subquery(query) do + from( + q in subquery(query), + select: q + ) + end + + defp common_where_limit_order(query, paging_options) do + query |> InternalTransaction.where_is_different_from_parent_transaction() |> InternalTransaction.where_block_number_is_not_null() |> page_internal_transaction(paging_options) @@ -205,9 +263,6 @@ defmodule Explorer.Chain do desc: it.transaction_index, desc: it.index ) - |> preload(transaction: :block) - |> join_associations(necessity_by_association) - |> Repo.all() end @doc """ @@ -2438,6 +2493,7 @@ defmodule Explorer.Chain do |> for_parent_transaction(hash) |> join_associations(necessity_by_association) |> where_transaction_has_multiple_internal_transactions() + |> InternalTransaction.where_is_different_from_parent_transaction() |> page_internal_transaction(paging_options) |> limit(^paging_options.page_size) |> order_by([internal_transaction], asc: internal_transaction.index) diff --git a/apps/explorer/lib/explorer/chain/internal_transaction.ex b/apps/explorer/lib/explorer/chain/internal_transaction.ex index 77f7d15da2..b293035284 100644 --- a/apps/explorer/lib/explorer/chain/internal_transaction.ex +++ b/apps/explorer/lib/explorer/chain/internal_transaction.ex @@ -491,6 +491,18 @@ defmodule Explorer.Chain.InternalTransaction do ) end + def where_address_fields_match(query, address_hash, :to_address_hash) do + where(query, [it], it.to_address_hash == ^address_hash) + end + + def where_address_fields_match(query, address_hash, :from_address_hash) do + where(query, [it], it.from_address_hash == ^address_hash) + end + + def where_address_fields_match(query, address_hash, :created_contract_address_hash) do + where(query, [it], it.created_contract_address_hash == ^address_hash) + end + def where_is_different_from_parent_transaction(query) do where( query, diff --git a/apps/explorer/lib/explorer/etherscan.ex b/apps/explorer/lib/explorer/etherscan.ex index 71ee5794f7..a828ee19cb 100644 --- a/apps/explorer/lib/explorer/etherscan.ex +++ b/apps/explorer/lib/explorer/etherscan.ex @@ -3,7 +3,7 @@ defmodule Explorer.Etherscan do The etherscan context. """ - import Ecto.Query, only: [from: 2, where: 3, or_where: 3] + import Ecto.Query, only: [from: 2, where: 3, or_where: 3, union: 2] alias Explorer.Etherscan.Logs alias Explorer.{Chain, Repo} @@ -97,6 +97,7 @@ defmodule Explorer.Etherscan do query |> Chain.where_transaction_has_multiple_internal_transactions() + |> InternalTransaction.where_is_different_from_parent_transaction() |> Repo.all() end @@ -120,27 +121,90 @@ defmodule Explorer.Etherscan do ) do options = Map.merge(@default_options, raw_options) - query = + direction = + case options do + %{filter_by: "to"} -> :to + %{filter_by: "from"} -> :from + _ -> nil + end + + consensus_blocks = from( - it in InternalTransaction, - inner_join: t in assoc(it, :transaction), - inner_join: b in assoc(t, :block), - order_by: [{^options.order_by_direction, t.block_number}], - limit: ^options.page_size, - offset: ^offset(options), - select: - merge(map(it, ^@internal_transaction_fields), %{ - block_timestamp: b.timestamp, - block_number: b.number - }) + b in Block, + where: b.consensus == true ) - query - |> Chain.where_transaction_has_multiple_internal_transactions() - |> where_address_match(address_hash, options) - |> where_start_block_match(options) - |> where_end_block_match(options) - |> Repo.all() + if direction == nil do + query = + from( + it in InternalTransaction, + inner_join: b in subquery(consensus_blocks), + on: it.block_number == b.number, + order_by: [ + {^options.order_by_direction, it.block_number}, + {:desc, it.transaction_index}, + {:desc, it.index} + ], + limit: ^options.page_size, + offset: ^offset(options), + select: + merge(map(it, ^@internal_transaction_fields), %{ + block_timestamp: b.timestamp, + block_number: b.number + }) + ) + + query_to_address_hash_wrapped = + query + |> InternalTransaction.where_address_fields_match(address_hash, :to_address_hash) + |> InternalTransaction.where_is_different_from_parent_transaction() + |> where_start_block_match(options) + |> where_end_block_match(options) + |> Chain.wrapped_union_subquery() + + query_from_address_hash_wrapped = + query + |> InternalTransaction.where_address_fields_match(address_hash, :from_address_hash) + |> InternalTransaction.where_is_different_from_parent_transaction() + |> where_start_block_match(options) + |> where_end_block_match(options) + |> Chain.wrapped_union_subquery() + + query_created_contract_address_hash_wrapped = + query + |> InternalTransaction.where_address_fields_match(address_hash, :created_contract_address_hash) + |> InternalTransaction.where_is_different_from_parent_transaction() + |> where_start_block_match(options) + |> where_end_block_match(options) + |> Chain.wrapped_union_subquery() + + query_to_address_hash_wrapped + |> union(^query_from_address_hash_wrapped) + |> union(^query_created_contract_address_hash_wrapped) + |> Repo.all() + else + query = + from( + it in InternalTransaction, + inner_join: t in assoc(it, :transaction), + inner_join: b in assoc(t, :block), + order_by: [{^options.order_by_direction, t.block_number}], + limit: ^options.page_size, + offset: ^offset(options), + select: + merge(map(it, ^@internal_transaction_fields), %{ + block_timestamp: b.timestamp, + block_number: b.number + }) + ) + + query + |> InternalTransaction.where_address_fields_match(address_hash, direction) + |> InternalTransaction.where_is_different_from_parent_transaction() + |> where_start_block_match(options) + |> where_end_block_match(options) + |> Repo.all() + end end @doc """ diff --git a/apps/explorer/priv/repo/migrations/20191203112646_internal_transactions_add_to_address_hash_index.exs b/apps/explorer/priv/repo/migrations/20191203112646_internal_transactions_add_to_address_hash_index.exs new file mode 100644 index 0000000000..f0ab878760 --- /dev/null +++ b/apps/explorer/priv/repo/migrations/20191203112646_internal_transactions_add_to_address_hash_index.exs @@ -0,0 +1,25 @@ +defmodule Explorer.Repo.Migrations.InternalTransactionsAddToAddressHashIndex do + use Ecto.Migration + + def change do + execute( + "CREATE INDEX IF NOT EXISTS internal_transactions_from_address_hash_partial_index on public.internal_transactions(from_address_hash, block_number DESC, transaction_index DESC, index DESC) WHERE (((type = 'call') AND (index > 0)) OR (type != 'call'));" + ) + + execute( + "CREATE INDEX IF NOT EXISTS internal_transactions_to_address_hash_partial_index on public.internal_transactions(to_address_hash, block_number DESC, transaction_index DESC, index DESC) WHERE (((type = 'call') AND (index > 0)) OR (type != 'call'));" + ) + + execute( + "CREATE INDEX IF NOT EXISTS internal_transactions_created_contract_address_hash_partial_index on public.internal_transactions(created_contract_address_hash, block_number DESC, transaction_index DESC, index DESC) WHERE (((type = 'call') AND (index > 0)) OR (type != 'call'));" + ) + + drop_if_exists( + index( + :internal_transactions, + [:to_address_hash, :from_address_hash, :created_contract_address_hash, :type, :index], + name: "internal_transactions_to_address_hash_from_address_hash_created" + ) + ) + end +end diff --git a/apps/explorer/test/explorer/chain_test.exs b/apps/explorer/test/explorer/chain_test.exs index 5aa8a6b1d2..913930e828 100644 --- a/apps/explorer/test/explorer/chain_test.exs +++ b/apps/explorer/test/explorer/chain_test.exs @@ -2225,7 +2225,8 @@ defmodule Explorer.ChainTest do results = [internal_transaction | _] = Chain.transaction_to_internal_transactions(transaction.hash) - assert 2 == length(results) + # excluding of internal transactions with type=call and index=0 + assert 1 == length(results) assert Enum.all?( results, @@ -2360,7 +2361,7 @@ defmodule Explorer.ChainTest do |> insert() |> with_block() - %InternalTransaction{transaction_hash: first_transaction_hash, index: first_index} = + %InternalTransaction{transaction_hash: _, index: _} = insert(:internal_transaction, transaction: transaction, index: 0, @@ -2381,7 +2382,8 @@ defmodule Explorer.ChainTest do |> Chain.transaction_to_internal_transactions() |> Enum.map(&{&1.transaction_hash, &1.index}) - assert [{first_transaction_hash, first_index}, {second_transaction_hash, second_index}] == result + # excluding of internal transactions with type=call and index=0 + assert [{second_transaction_hash, second_index}] == result end test "pages by index" do @@ -2390,7 +2392,7 @@ defmodule Explorer.ChainTest do |> insert() |> with_block() - %InternalTransaction{transaction_hash: first_transaction_hash, index: first_index} = + %InternalTransaction{transaction_hash: _, index: _} = insert(:internal_transaction, transaction: transaction, index: 0, @@ -2406,19 +2408,27 @@ defmodule Explorer.ChainTest do transaction_index: transaction.index ) - assert [{first_transaction_hash, first_index}, {second_transaction_hash, second_index}] == + %InternalTransaction{transaction_hash: third_transaction_hash, index: third_index} = + insert(:internal_transaction, + transaction: transaction, + index: 2, + block_number: transaction.block_number, + transaction_index: transaction.index + ) + + assert [{second_transaction_hash, second_index}, {third_transaction_hash, third_index}] == transaction.hash |> Chain.transaction_to_internal_transactions(paging_options: %PagingOptions{key: {-1}, page_size: 2}) |> Enum.map(&{&1.transaction_hash, &1.index}) - assert [{first_transaction_hash, first_index}] == + assert [{second_transaction_hash, second_index}] == transaction.hash |> Chain.transaction_to_internal_transactions(paging_options: %PagingOptions{key: {-1}, page_size: 1}) |> Enum.map(&{&1.transaction_hash, &1.index}) - assert [{second_transaction_hash, second_index}] == + assert [{third_transaction_hash, third_index}] == transaction.hash - |> Chain.transaction_to_internal_transactions(paging_options: %PagingOptions{key: {0}, page_size: 2}) + |> Chain.transaction_to_internal_transactions(paging_options: %PagingOptions{key: {1}, page_size: 2}) |> Enum.map(&{&1.transaction_hash, &1.index}) end end diff --git a/apps/explorer/test/explorer/etherscan_test.exs b/apps/explorer/test/explorer/etherscan_test.exs index 5d1cf7fcb1..e2e9f0c273 100644 --- a/apps/explorer/test/explorer/etherscan_test.exs +++ b/apps/explorer/test/explorer/etherscan_test.exs @@ -533,7 +533,8 @@ defmodule Explorer.EtherscanTest do found_internal_transactions = Etherscan.list_internal_transactions(transaction.hash) - assert length(found_internal_transactions) == 3 + # excluding of internal transactions with type=call and index=0 + assert length(found_internal_transactions) == 2 end test "only returns internal transactions that belong to the transaction" do @@ -571,7 +572,8 @@ defmodule Explorer.EtherscanTest do internal_transactions1 = Etherscan.list_internal_transactions(transaction1.hash) - assert length(internal_transactions1) == 2 + # excluding of internal transactions with type=call and index=0 + assert length(internal_transactions1) == 1 internal_transactions2 = Etherscan.list_internal_transactions(transaction2.hash) @@ -659,7 +661,7 @@ defmodule Explorer.EtherscanTest do |> insert() |> with_block() - for index <- 0..2 do + for index <- 0..3 do internal_transaction_details = %{ transaction: transaction, index: index, @@ -719,7 +721,8 @@ defmodule Explorer.EtherscanTest do internal_transactions1 = Etherscan.list_internal_transactions(address1.hash) - assert length(internal_transactions1) == 3 + # excluding of internal transactions with type=call and index=0 + assert length(internal_transactions1) == 2 internal_transactions2 = Etherscan.list_internal_transactions(address2.hash) @@ -734,7 +737,7 @@ defmodule Explorer.EtherscanTest do |> insert() |> with_block() - for index <- 0..2 do + for index <- 0..3 do internal_transaction_details = %{ transaction: transaction, index: index, @@ -795,7 +798,8 @@ defmodule Explorer.EtherscanTest do expected_block_numbers = [second_block.number, third_block.number] - assert length(found_internal_transactions) == 4 + # excluding of internal transactions with type=call and index=0 + assert length(found_internal_transactions) == 2 for internal_transaction <- found_internal_transactions do assert internal_transaction.block_number in expected_block_numbers From 6ec1ac1a7e54e2d24afedd30347094353a69e66a Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Sun, 8 Dec 2019 11:55:33 +0300 Subject: [PATCH 4/8] Modify logs query --- apps/explorer/lib/explorer/etherscan/logs.ex | 57 +++++++++++++------- 1 file changed, 38 insertions(+), 19 deletions(-) diff --git a/apps/explorer/lib/explorer/etherscan/logs.ex b/apps/explorer/lib/explorer/etherscan/logs.ex index 336eae106b..066daf74ef 100644 --- a/apps/explorer/lib/explorer/etherscan/logs.ex +++ b/apps/explorer/lib/explorer/etherscan/logs.ex @@ -5,8 +5,9 @@ defmodule Explorer.Etherscan.Logs do """ - import Ecto.Query, only: [from: 2, where: 3, subquery: 1, order_by: 3] + import Ecto.Query, only: [from: 2, where: 3, subquery: 1, order_by: 3, union: 2] + alias Explorer.Chain alias Explorer.Chain.{Block, InternalTransaction, Log, Transaction} alias Explorer.Repo @@ -78,25 +79,25 @@ defmodule Explorer.Etherscan.Logs do logs_query = where_topic_match(Log, prepared_filter) + query_to_address_hash_wrapped = + logs_query + |> internal_transaction_query(:to_address_hash, prepared_filter, address_hash) + |> Chain.wrapped_union_subquery() + + query_from_address_hash_wrapped = + logs_query + |> internal_transaction_query(:from_address_hash, prepared_filter, address_hash) + |> Chain.wrapped_union_subquery() + + query_created_contract_address_hash_wrapped = + logs_query + |> internal_transaction_query(:created_contract_address_hash, prepared_filter, address_hash) + |> Chain.wrapped_union_subquery() + internal_transaction_log_query = - from(internal_transaction in InternalTransaction, - join: transaction in assoc(internal_transaction, :transaction), - join: log in ^logs_query, - on: log.transaction_hash == internal_transaction.transaction_hash, - where: internal_transaction.block_number >= ^prepared_filter.from_block, - where: internal_transaction.block_number <= ^prepared_filter.to_block, - where: - internal_transaction.to_address_hash == ^address_hash or - internal_transaction.from_address_hash == ^address_hash or - internal_transaction.created_contract_address_hash == ^address_hash, - select: - merge(map(log, ^@log_fields), %{ - gas_price: transaction.gas_price, - gas_used: transaction.gas_used, - transaction_index: transaction.index, - block_number: transaction.block_number - }) - ) + query_to_address_hash_wrapped + |> union(^query_from_address_hash_wrapped) + |> union(^query_created_contract_address_hash_wrapped) all_transaction_logs_query = from(transaction in Transaction, @@ -256,4 +257,22 @@ defmodule Explorer.Etherscan.Logs do data.transaction_index >= ^transaction_index ) end + + defp internal_transaction_query(logs_query, direction, prepared_filter, address_hash) do + from(internal_transaction in InternalTransaction, + join: transaction in assoc(internal_transaction, :transaction), + join: log in ^logs_query, + on: log.transaction_hash == internal_transaction.transaction_hash, + where: internal_transaction.block_number >= ^prepared_filter.from_block, + where: internal_transaction.block_number <= ^prepared_filter.to_block, + select: + merge(map(log, ^@log_fields), %{ + gas_price: transaction.gas_price, + gas_used: transaction.gas_used, + transaction_index: transaction.index, + block_number: transaction.block_number + }) + ) + |> InternalTransaction.where_address_fields_match(address_hash, direction) + end end From e167d2720585bf4a9de6d8dc429564ffbbe37080 Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Wed, 11 Dec 2019 19:50:53 +0300 Subject: [PATCH 5/8] Fix performance of address page even without limiting of rewards to retrieve --- apps/explorer/lib/explorer/chain.ex | 71 ++++++++++++++++--- .../lib/explorer/chain/block/reward.ex | 38 +++++++--- .../test/explorer/chain/cache/uncles_test.exs | 1 - .../chain/import/runner/blocks_test.exs | 2 +- apps/explorer/test/explorer/chain_test.exs | 31 +++++++- 5 files changed, 120 insertions(+), 23 deletions(-) diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index 9576eca1a6..85e385a43b 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -261,10 +261,10 @@ defmodule Explorer.Chain do paging_options = Keyword.get(options, :paging_options, @default_paging_options) if Application.get_env(:block_scout_web, BlockScoutWeb.Chain)[:has_emission_funds] do + blocks_range = address_to_transactions_tasks_range_of_blocks(address_hash, options) + rewards_task = - Task.async(fn -> - Reward.fetch_emission_rewards_tuples(address_hash, paging_options) - end) + Task.async(fn -> Reward.fetch_emission_rewards_tuples(address_hash, paging_options, blocks_range) end) [rewards_task | address_to_transactions_tasks(address_hash, options)] |> wait_for_address_transactions() @@ -303,21 +303,72 @@ defmodule Explorer.Chain do |> Enum.take(paging_options.page_size) end + defp address_to_transactions_tasks_query(options) do + options + |> Keyword.get(:paging_options, @default_paging_options) + |> fetch_transactions() + end + defp address_to_transactions_tasks(address_hash, options) do - paging_options = Keyword.get(options, :paging_options, @default_paging_options) direction = Keyword.get(options, :direction) necessity_by_association = Keyword.get(options, :necessity_by_association, %{}) - base_query = - paging_options - |> fetch_transactions() - |> join_associations(necessity_by_association) - - base_query + options + |> address_to_transactions_tasks_query() + |> join_associations(necessity_by_association) |> Transaction.matching_address_queries_list(direction, address_hash) |> Enum.map(fn query -> Task.async(fn -> Repo.all(query) end) end) end + defp address_to_transactions_tasks_range_of_blocks(address_hash, options) do + direction = Keyword.get(options, :direction) + + extremums_list = + options + |> address_to_transactions_tasks_query() + |> Transaction.matching_address_queries_list(direction, address_hash) + |> Enum.map(fn query -> + max_query = + from( + q in subquery(query), + select: %{min_block_number: min(q.block_number), max_block_number: max(q.block_number)} + ) + + max_query + |> Repo.one!() + end) + + extremums_list + |> Enum.reduce(%{min_block_number: nil, max_block_number: 0}, fn %{ + min_block_number: min_number, + max_block_number: max_number + }, + extremums_result -> + current_min_number = Map.get(extremums_result, :min_block_number) + current_max_number = Map.get(extremums_result, :max_block_number) + + extremums_result = + if is_number(current_min_number) do + if is_number(min_number) and min_number > 0 and min_number < current_min_number do + extremums_result + |> Map.put(:min_block_number, min_number) + else + extremums_result + end + else + extremums_result + |> Map.put(:min_block_number, min_number) + end + + if is_number(max_number) and max_number > 0 and max_number > current_max_number do + extremums_result + |> Map.put(:max_block_number, max_number) + else + extremums_result + end + end) + end + defp wait_for_address_transactions(tasks) do tasks |> Task.yield_many(:timer.seconds(20)) diff --git a/apps/explorer/lib/explorer/chain/block/reward.ex b/apps/explorer/lib/explorer/chain/block/reward.ex index aa73ab4cb1..0a33de7153 100644 --- a/apps/explorer/lib/explorer/chain/block/reward.ex +++ b/apps/explorer/lib/explorer/chain/block/reward.ex @@ -68,8 +68,10 @@ defmodule Explorer.Chain.Block.Reward do Returns a list of tuples representing rewards by the EmissionFunds on POA chains. The tuples have the format {EmissionFunds, Validator} """ - @spec fetch_emission_rewards_tuples(Hash.Address.t(), PagingOptions.t()) :: [{t(), t()}] - def fetch_emission_rewards_tuples(address_hash, paging_options) do + def fetch_emission_rewards_tuples(address_hash, paging_options, %{ + min_block_number: min_block_number, + max_block_number: max_block_number + }) do address_rewards = __MODULE__ |> join_associations() @@ -77,6 +79,7 @@ defmodule Explorer.Chain.Block.Reward do |> limit(^paging_options.page_size) |> order_by([_, block], desc: block.number) |> where([reward], reward.address_hash == ^address_hash) + |> address_rewards_blocks_ranges_clause(min_block_number, max_block_number, paging_options) |> Repo.all() case List.first(address_rewards) do @@ -112,15 +115,30 @@ defmodule Explorer.Chain.Block.Reward do end defp join_associations(query) do - last_blocks_query = - from(block in Block, - limit: 2000, - order_by: [desc: block.number] - ) - query |> preload(:address) - |> join(:inner, [reward], block in subquery(last_blocks_query), on: block.hash == reward.block_hash) - |> preload(block: ^last_blocks_query) + |> join(:inner, [reward], block in assoc(reward, :block)) + |> preload(:block) + end + + defp address_rewards_blocks_ranges_clause(query, min_block_number, max_block_number, paging_options) do + if is_number(min_block_number) and max_block_number > 0 and min_block_number > 0 do + cond do + paging_options.page_number == 1 -> + query + |> where([_, block], block.number >= ^min_block_number) + + min_block_number == max_block_number -> + query + |> where([_, block], block.number == ^min_block_number) + + true -> + query + |> where([_, block], block.number >= ^min_block_number) + |> where([_, block], block.number <= ^max_block_number) + end + else + query + end end end diff --git a/apps/explorer/test/explorer/chain/cache/uncles_test.exs b/apps/explorer/test/explorer/chain/cache/uncles_test.exs index c8984d34ea..230889ca77 100644 --- a/apps/explorer/test/explorer/chain/cache/uncles_test.exs +++ b/apps/explorer/test/explorer/chain/cache/uncles_test.exs @@ -2,7 +2,6 @@ defmodule Explorer.Chain.Cache.UnclesTest do use Explorer.DataCase alias Explorer.Chain.Cache.Uncles - alias Explorer.Repo setup do Supervisor.terminate_child(Explorer.Supervisor, Uncles.child_id()) diff --git a/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs b/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs index c47204e749..44f7388c56 100644 --- a/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs +++ b/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs @@ -384,7 +384,7 @@ defmodule Explorer.Chain.Import.Runner.BlocksTest do end test "removes duplicate blocks (by hash) before inserting", - %{consensus_block: %{number: block_number, hash: block_hash, miner_hash: miner_hash}, options: options} do + %{consensus_block: %{number: _, hash: block_hash, miner_hash: miner_hash}, options: options} do new_block = params_for(:block, miner_hash: miner_hash, consensus: true) %Ecto.Changeset{valid?: true, changes: block_changes} = Block.changeset(%Block{}, new_block) diff --git a/apps/explorer/test/explorer/chain_test.exs b/apps/explorer/test/explorer/chain_test.exs index 5aa8a6b1d2..702b96bbb9 100644 --- a/apps/explorer/test/explorer/chain_test.exs +++ b/apps/explorer/test/explorer/chain_test.exs @@ -534,7 +534,7 @@ defmodule Explorer.ChainTest do :transaction |> insert(from_address: block.miner) - |> with_block() + |> with_block(block) |> Repo.preload(:token_transfers) assert [_, {_, _}] = Chain.address_to_transactions_with_rewards(block.miner.hash, direction: :from) @@ -542,6 +542,35 @@ defmodule Explorer.ChainTest do Application.put_env(:block_scout_web, BlockScoutWeb.Chain, has_emission_funds: false) end + test "with transactions if rewards are not in the range of blocks" do + Application.put_env(:block_scout_web, BlockScoutWeb.Chain, has_emission_funds: true) + + block = insert(:block) + + insert( + :reward, + address_hash: block.miner_hash, + block_hash: block.hash, + address_type: :validator + ) + + insert( + :reward, + address_hash: block.miner_hash, + block_hash: block.hash, + address_type: :emission_funds + ) + + :transaction + |> insert(from_address: block.miner) + |> with_block() + |> Repo.preload(:token_transfers) + + assert [_] = Chain.address_to_transactions_with_rewards(block.miner.hash, direction: :from) + + Application.put_env(:block_scout_web, BlockScoutWeb.Chain, has_emission_funds: false) + end + test "with emissions rewards, but feature disabled" do Application.put_env(:block_scout_web, BlockScoutWeb.Chain, has_emission_funds: false) From 98374fba32017cca18e6c01b913ea5f3d6aa6afb Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Fri, 3 Jan 2020 22:33:40 +0300 Subject: [PATCH 6/8] RSK release 1.2.0 breaking changes support --- CHANGELOG.md | 1 + apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/block.ex | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ea21b56fb..ef8497af0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ - [#2918](https://github.com/poanetwork/blockscout/pull/2918) - Add tokenID for tokentx API action ### Fixes +- [#2934](https://github.com/poanetwork/blockscout/pull/2934) - RSK release 1.2.0 breaking changes support - [#2915](https://github.com/poanetwork/blockscout/pull/2915) - Speedup of blocks_without_reward_query - [#2914](https://github.com/poanetwork/blockscout/pull/2914) - Reduce execution time of stream_unfetched_token_instances query - [#2906](https://github.com/poanetwork/blockscout/pull/2906) - fix address sum cache diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/block.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/block.ex index 1c52a6e6c9..de83b3601e 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/block.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/block.ex @@ -437,7 +437,8 @@ defmodule EthereumJSONRPC.Block do end defp entry_to_elixir({key, quantity}) - when key in ~w(difficulty gasLimit gasUsed minimumGasPrice number size totalDifficulty) and not is_nil(quantity) do + when key in ~w(difficulty gasLimit gasUsed minimumGasPrice number size totalDifficulty paidFees) and + not is_nil(quantity) do {key, quantity_to_integer(quantity)} end @@ -451,7 +452,7 @@ defmodule EthereumJSONRPC.Block do # hash format defp entry_to_elixir({key, _} = entry) when key in ~w(author extraData hash logsBloom miner mixHash nonce parentHash receiptsRoot sealFields sha3Uncles - signature stateRoot step transactionsRoot uncles), + signature stateRoot step transactionsRoot uncles bitcoinMergedMiningCoinbaseTransaction bitcoinMergedMiningHeader bitcoinMergedMiningMerkleProof hashForMergedMining), do: entry defp entry_to_elixir({"timestamp" = key, timestamp}) do From 2271bc69efb96a5e1ae9e8bef438a134adfb8a9a Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Sat, 4 Jan 2020 23:20:48 +0300 Subject: [PATCH 7/8] Fix tests --- apps/explorer/lib/explorer/etherscan/logs.ex | 34 +++++++++++--------- apps/explorer/test/explorer/chain_test.exs | 2 ++ 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/apps/explorer/lib/explorer/etherscan/logs.ex b/apps/explorer/lib/explorer/etherscan/logs.ex index b4423981d7..35dbfda1ba 100644 --- a/apps/explorer/lib/explorer/etherscan/logs.ex +++ b/apps/explorer/lib/explorer/etherscan/logs.ex @@ -7,9 +7,8 @@ defmodule Explorer.Etherscan.Logs do import Ecto.Query, only: [from: 2, where: 3, subquery: 1, order_by: 3, union: 2] - alias Explorer.Chain + alias Explorer.{Chain, Repo} alias Explorer.Chain.{Block, InternalTransaction, Log, Transaction} - alias Explorer.Repo @base_filter %{ from_block: nil, @@ -259,20 +258,23 @@ defmodule Explorer.Etherscan.Logs do end defp internal_transaction_query(logs_query, direction, prepared_filter, address_hash) do - from(internal_transaction in InternalTransaction.where_nonpending_block(), - join: transaction in assoc(internal_transaction, :transaction), - join: log in ^logs_query, - on: log.transaction_hash == internal_transaction.transaction_hash, - where: internal_transaction.block_number >= ^prepared_filter.from_block, - where: internal_transaction.block_number <= ^prepared_filter.to_block, - select: - merge(map(log, ^@log_fields), %{ - gas_price: transaction.gas_price, - gas_used: transaction.gas_used, - transaction_index: transaction.index, - block_number: transaction.block_number - }) - ) + query = + from(internal_transaction in InternalTransaction.where_nonpending_block(), + join: transaction in assoc(internal_transaction, :transaction), + join: log in ^logs_query, + on: log.transaction_hash == internal_transaction.transaction_hash, + where: internal_transaction.block_number >= ^prepared_filter.from_block, + where: internal_transaction.block_number <= ^prepared_filter.to_block, + select: + merge(map(log, ^@log_fields), %{ + gas_price: transaction.gas_price, + gas_used: transaction.gas_used, + transaction_index: transaction.index, + block_number: transaction.block_number + }) + ) + + query |> InternalTransaction.where_address_fields_match(address_hash, direction) end end diff --git a/apps/explorer/test/explorer/chain_test.exs b/apps/explorer/test/explorer/chain_test.exs index 52ca4fe016..b37edbc36e 100644 --- a/apps/explorer/test/explorer/chain_test.exs +++ b/apps/explorer/test/explorer/chain_test.exs @@ -2598,6 +2598,8 @@ defmodule Explorer.ChainTest do transaction: transaction, index: 2, block_number: transaction.block_number, + block_hash: transaction.block_hash, + block_index: 2, transaction_index: transaction.index ) From 7466322f944f12445e915268cf8d69ab606fe94e Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Sun, 5 Jan 2020 10:55:43 +0300 Subject: [PATCH 8/8] Improve DB migration --- CHANGELOG.md | 2 +- ...054_add_pending_internal_txs_operation.exs | 35 ++++++++++++++++--- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2511d7be3..b13d14555e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ ## Current ### Features -- [#2835](https://github.com/poanetwork/blockscout/pull/2835), [#2871](https://github.com/poanetwork/blockscout/pull/2871), [#2872](https://github.com/poanetwork/blockscout/pull/2872), [#2886](https://github.com/poanetwork/blockscout/pull/2886), [#2925](https://github.com/poanetwork/blockscout/pull/2925) - Add "block_hash" to logs, token_transfers and internal transactions and "pending blocks operations" approach +- [#2835](https://github.com/poanetwork/blockscout/pull/2835), [#2871](https://github.com/poanetwork/blockscout/pull/2871), [#2872](https://github.com/poanetwork/blockscout/pull/2872), [#2886](https://github.com/poanetwork/blockscout/pull/2886), [#2925](https://github.com/poanetwork/blockscout/pull/2925), [#2936](https://github.com/poanetwork/blockscout/pull/2936) - Add "block_hash" to logs, token_transfers and internal transactions and "pending blocks operations" approach - [#2926](https://github.com/poanetwork/blockscout/pull/2926) - API endpoint: sum balances except burnt address - [#2918](https://github.com/poanetwork/blockscout/pull/2918) - Add tokenID for tokentx API action explicitly diff --git a/apps/explorer/priv/repo/migrations/20191018140054_add_pending_internal_txs_operation.exs b/apps/explorer/priv/repo/migrations/20191018140054_add_pending_internal_txs_operation.exs index d603d1e853..41af522005 100644 --- a/apps/explorer/priv/repo/migrations/20191018140054_add_pending_internal_txs_operation.exs +++ b/apps/explorer/priv/repo/migrations/20191018140054_add_pending_internal_txs_operation.exs @@ -60,11 +60,37 @@ defmodule Explorer.Repo.Migrations.AddPendingInternalTxsOperation do DECLARE duplicates_count INTEGER := 0; blocks_scanned INTEGER := 0; + int_txs_count INTEGER := 0; temprow RECORD; BEGIN - FOR temprow IN - SELECT number, hash FROM blocks - LOOP + SELECT COUNT(*) INTO int_txs_count FROM internal_transactions; + IF int_txs_count < 10000000 THEN + + FOR temprow IN + SELECT block_hash FROM internal_transactions + GROUP BY block_hash, block_index HAVING COUNT(*) > 1 + LOOP + duplicates_count := duplicates_count + 1; + RAISE NOTICE '% duplicates, blocks scanned %, block #%, block hash is %', duplicates_count, blocks_scanned, temprow.number , temprow.hash; + + IF NOT EXISTS ( + SELECT 1 FROM pending_block_operations + WHERE block_hash = temprow.block_hash + ) THEN + INSERT INTO pending_block_operations + (block_hash, inserted_at, updated_at, fetch_internal_transactions) + SELECT b.hash, now(), now(), TRUE FROM blocks b + WHERE b.hash = temprow.block_hash; + END IF; + + DELETE FROM internal_transactions + WHERE block_hash = temprow.block_hash; + + RAISE NOTICE 'DELETED'; + END LOOP; + + ELSE + FOR temprow IN SELECT number, hash FROM blocks LOOP blocks_scanned := blocks_scanned + 1; IF EXISTS ( SELECT 1 FROM transactions WHERE block_hash = temprow.hash @@ -93,7 +119,8 @@ defmodule Explorer.Repo.Migrations.AddPendingInternalTxsOperation do RAISE NOTICE 'DELETED'; END IF; END IF; - END LOOP; + END LOOP; + END IF; RAISE NOTICE 'SCRIPT FINISHED'; END $$; """)