Reorganize queries and indexes for internal_transactions table

pull/2910/head
Victor Baranov 5 years ago
parent d7d242e44b
commit d3e258c431
  1. 2
      CHANGELOG.md
  2. 8
      apps/block_scout_web/test/block_scout_web/controllers/api/rpc/address_controller_test.exs
  3. 3
      apps/block_scout_web/test/block_scout_web/controllers/transaction_internal_transaction_controller_test.exs
  4. 66
      apps/explorer/lib/explorer/chain.ex
  5. 12
      apps/explorer/lib/explorer/chain/internal_transaction.ex
  6. 102
      apps/explorer/lib/explorer/etherscan.ex
  7. 25
      apps/explorer/priv/repo/migrations/20191203112646_internal_transactions_add_to_address_hash_index.exs
  8. 26
      apps/explorer/test/explorer/chain_test.exs
  9. 16
      apps/explorer/test/explorer/etherscan_test.exs

@ -4,6 +4,8 @@
### Fixes
- [2910](https://github.com/poanetwork/blockscout/pull/2910) - Reorganize queries and indexes for internal_transactions table
### Chore
- [#2896](https://github.com/poanetwork/blockscout/pull/2896) - Disable Parity websockets tests

@ -1606,7 +1606,7 @@ defmodule BlockScoutWeb.API.RPC.AddressControllerTest do
internal_transaction =
:internal_transaction_create
|> insert(transaction: transaction, index: 0, from_address: address)
|> insert(transaction: transaction, index: 0, from_address: address, block_number: block.number)
|> with_contract_creation(contract_address)
params = %{
@ -1658,7 +1658,8 @@ defmodule BlockScoutWeb.API.RPC.AddressControllerTest do
transaction: transaction,
index: 0,
type: :reward,
error: "some error"
error: "some error",
block_number: transaction.block_number
]
insert(:internal_transaction_create, internal_transaction_details)
@ -1693,7 +1694,8 @@ defmodule BlockScoutWeb.API.RPC.AddressControllerTest do
internal_transaction_details = %{
from_address: address,
transaction: transaction,
index: index
index: index,
block_number: transaction.block_number
}
insert(:internal_transaction_create, internal_transaction_details)

@ -64,7 +64,8 @@ defmodule BlockScoutWeb.TransactionInternalTransactionControllerTest do
assert json_response(conn, 200)
assert Enum.count(items) == 2
# excluding of internal transactions with type=call and index=0
assert Enum.count(items) == 1
end
test "includes USD exchange rate value for address in assigns", %{conn: conn} do

@ -15,6 +15,7 @@ defmodule Explorer.Chain do
preload: 2,
select: 2,
subquery: 1,
union: 2,
union_all: 2,
where: 2,
where: 3
@ -193,8 +194,65 @@ defmodule Explorer.Chain do
direction = Keyword.get(options, :direction)
paging_options = Keyword.get(options, :paging_options, @default_paging_options)
InternalTransaction
|> InternalTransaction.where_address_fields_match(hash, direction)
if direction == nil do
query_to_address_hash_wrapped =
InternalTransaction
|> InternalTransaction.where_address_fields_match(hash, :to_address_hash)
|> common_where_limit_order(paging_options)
|> wrapped_union_subquery()
query_from_address_hash_wrapped =
InternalTransaction
|> InternalTransaction.where_address_fields_match(hash, :from_address_hash)
|> common_where_limit_order(paging_options)
|> wrapped_union_subquery()
query_created_contract_address_hash_wrapped =
InternalTransaction
|> InternalTransaction.where_address_fields_match(hash, :created_contract_address_hash)
|> common_where_limit_order(paging_options)
|> wrapped_union_subquery()
full_query =
query_to_address_hash_wrapped
|> union(^query_from_address_hash_wrapped)
|> union(^query_created_contract_address_hash_wrapped)
full_wrapped_query =
from(
q in subquery(full_query),
select: q
)
full_wrapped_query
|> order_by(
[q],
desc: q.block_number,
desc: q.transaction_index,
desc: q.index
)
|> preload(transaction: :block)
|> join_associations(necessity_by_association)
|> Repo.all()
else
InternalTransaction
|> InternalTransaction.where_address_fields_match(hash, direction)
|> common_where_limit_order(paging_options)
|> preload(transaction: :block)
|> join_associations(necessity_by_association)
|> Repo.all()
end
end
def wrapped_union_subquery(query) do
from(
q in subquery(query),
select: q
)
end
defp common_where_limit_order(query, paging_options) do
query
|> InternalTransaction.where_is_different_from_parent_transaction()
|> InternalTransaction.where_block_number_is_not_null()
|> page_internal_transaction(paging_options)
@ -205,9 +263,6 @@ defmodule Explorer.Chain do
desc: it.transaction_index,
desc: it.index
)
|> preload(transaction: :block)
|> join_associations(necessity_by_association)
|> Repo.all()
end
@doc """
@ -2438,6 +2493,7 @@ defmodule Explorer.Chain do
|> for_parent_transaction(hash)
|> join_associations(necessity_by_association)
|> where_transaction_has_multiple_internal_transactions()
|> InternalTransaction.where_is_different_from_parent_transaction()
|> page_internal_transaction(paging_options)
|> limit(^paging_options.page_size)
|> order_by([internal_transaction], asc: internal_transaction.index)

@ -491,6 +491,18 @@ defmodule Explorer.Chain.InternalTransaction do
)
end
def where_address_fields_match(query, address_hash, :to_address_hash) do
where(query, [it], it.to_address_hash == ^address_hash)
end
def where_address_fields_match(query, address_hash, :from_address_hash) do
where(query, [it], it.from_address_hash == ^address_hash)
end
def where_address_fields_match(query, address_hash, :created_contract_address_hash) do
where(query, [it], it.created_contract_address_hash == ^address_hash)
end
def where_is_different_from_parent_transaction(query) do
where(
query,

@ -3,7 +3,7 @@ defmodule Explorer.Etherscan do
The etherscan context.
"""
import Ecto.Query, only: [from: 2, where: 3, or_where: 3]
import Ecto.Query, only: [from: 2, where: 3, or_where: 3, union: 2]
alias Explorer.Etherscan.Logs
alias Explorer.{Chain, Repo}
@ -97,6 +97,7 @@ defmodule Explorer.Etherscan do
query
|> Chain.where_transaction_has_multiple_internal_transactions()
|> InternalTransaction.where_is_different_from_parent_transaction()
|> Repo.all()
end
@ -120,27 +121,90 @@ defmodule Explorer.Etherscan do
) do
options = Map.merge(@default_options, raw_options)
query =
direction =
case options do
%{filter_by: "to"} -> :to
%{filter_by: "from"} -> :from
_ -> nil
end
consensus_blocks =
from(
it in InternalTransaction,
inner_join: t in assoc(it, :transaction),
inner_join: b in assoc(t, :block),
order_by: [{^options.order_by_direction, t.block_number}],
limit: ^options.page_size,
offset: ^offset(options),
select:
merge(map(it, ^@internal_transaction_fields), %{
block_timestamp: b.timestamp,
block_number: b.number
})
b in Block,
where: b.consensus == true
)
query
|> Chain.where_transaction_has_multiple_internal_transactions()
|> where_address_match(address_hash, options)
|> where_start_block_match(options)
|> where_end_block_match(options)
|> Repo.all()
if direction == nil do
query =
from(
it in InternalTransaction,
inner_join: b in subquery(consensus_blocks),
on: it.block_number == b.number,
order_by: [
{^options.order_by_direction, it.block_number},
{:desc, it.transaction_index},
{:desc, it.index}
],
limit: ^options.page_size,
offset: ^offset(options),
select:
merge(map(it, ^@internal_transaction_fields), %{
block_timestamp: b.timestamp,
block_number: b.number
})
)
query_to_address_hash_wrapped =
query
|> InternalTransaction.where_address_fields_match(address_hash, :to_address_hash)
|> InternalTransaction.where_is_different_from_parent_transaction()
|> where_start_block_match(options)
|> where_end_block_match(options)
|> Chain.wrapped_union_subquery()
query_from_address_hash_wrapped =
query
|> InternalTransaction.where_address_fields_match(address_hash, :from_address_hash)
|> InternalTransaction.where_is_different_from_parent_transaction()
|> where_start_block_match(options)
|> where_end_block_match(options)
|> Chain.wrapped_union_subquery()
query_created_contract_address_hash_wrapped =
query
|> InternalTransaction.where_address_fields_match(address_hash, :created_contract_address_hash)
|> InternalTransaction.where_is_different_from_parent_transaction()
|> where_start_block_match(options)
|> where_end_block_match(options)
|> Chain.wrapped_union_subquery()
query_to_address_hash_wrapped
|> union(^query_from_address_hash_wrapped)
|> union(^query_created_contract_address_hash_wrapped)
|> Repo.all()
else
query =
from(
it in InternalTransaction,
inner_join: t in assoc(it, :transaction),
inner_join: b in assoc(t, :block),
order_by: [{^options.order_by_direction, t.block_number}],
limit: ^options.page_size,
offset: ^offset(options),
select:
merge(map(it, ^@internal_transaction_fields), %{
block_timestamp: b.timestamp,
block_number: b.number
})
)
query
|> InternalTransaction.where_address_fields_match(address_hash, direction)
|> InternalTransaction.where_is_different_from_parent_transaction()
|> where_start_block_match(options)
|> where_end_block_match(options)
|> Repo.all()
end
end
@doc """

@ -0,0 +1,25 @@
defmodule Explorer.Repo.Migrations.InternalTransactionsAddToAddressHashIndex do
use Ecto.Migration
def change do
execute(
"CREATE INDEX IF NOT EXISTS internal_transactions_from_address_hash_partial_index on public.internal_transactions(from_address_hash, block_number DESC, transaction_index DESC, index DESC) WHERE (((type = 'call') AND (index > 0)) OR (type != 'call'));"
)
execute(
"CREATE INDEX IF NOT EXISTS internal_transactions_to_address_hash_partial_index on public.internal_transactions(to_address_hash, block_number DESC, transaction_index DESC, index DESC) WHERE (((type = 'call') AND (index > 0)) OR (type != 'call'));"
)
execute(
"CREATE INDEX IF NOT EXISTS internal_transactions_created_contract_address_hash_partial_index on public.internal_transactions(created_contract_address_hash, block_number DESC, transaction_index DESC, index DESC) WHERE (((type = 'call') AND (index > 0)) OR (type != 'call'));"
)
drop_if_exists(
index(
:internal_transactions,
[:to_address_hash, :from_address_hash, :created_contract_address_hash, :type, :index],
name: "internal_transactions_to_address_hash_from_address_hash_created"
)
)
end
end

@ -2225,7 +2225,8 @@ defmodule Explorer.ChainTest do
results = [internal_transaction | _] = Chain.transaction_to_internal_transactions(transaction.hash)
assert 2 == length(results)
# excluding of internal transactions with type=call and index=0
assert 1 == length(results)
assert Enum.all?(
results,
@ -2360,7 +2361,7 @@ defmodule Explorer.ChainTest do
|> insert()
|> with_block()
%InternalTransaction{transaction_hash: first_transaction_hash, index: first_index} =
%InternalTransaction{transaction_hash: _, index: _} =
insert(:internal_transaction,
transaction: transaction,
index: 0,
@ -2381,7 +2382,8 @@ defmodule Explorer.ChainTest do
|> Chain.transaction_to_internal_transactions()
|> Enum.map(&{&1.transaction_hash, &1.index})
assert [{first_transaction_hash, first_index}, {second_transaction_hash, second_index}] == result
# excluding of internal transactions with type=call and index=0
assert [{second_transaction_hash, second_index}] == result
end
test "pages by index" do
@ -2390,7 +2392,7 @@ defmodule Explorer.ChainTest do
|> insert()
|> with_block()
%InternalTransaction{transaction_hash: first_transaction_hash, index: first_index} =
%InternalTransaction{transaction_hash: _, index: _} =
insert(:internal_transaction,
transaction: transaction,
index: 0,
@ -2406,19 +2408,27 @@ defmodule Explorer.ChainTest do
transaction_index: transaction.index
)
assert [{first_transaction_hash, first_index}, {second_transaction_hash, second_index}] ==
%InternalTransaction{transaction_hash: third_transaction_hash, index: third_index} =
insert(:internal_transaction,
transaction: transaction,
index: 2,
block_number: transaction.block_number,
transaction_index: transaction.index
)
assert [{second_transaction_hash, second_index}, {third_transaction_hash, third_index}] ==
transaction.hash
|> Chain.transaction_to_internal_transactions(paging_options: %PagingOptions{key: {-1}, page_size: 2})
|> Enum.map(&{&1.transaction_hash, &1.index})
assert [{first_transaction_hash, first_index}] ==
assert [{second_transaction_hash, second_index}] ==
transaction.hash
|> Chain.transaction_to_internal_transactions(paging_options: %PagingOptions{key: {-1}, page_size: 1})
|> Enum.map(&{&1.transaction_hash, &1.index})
assert [{second_transaction_hash, second_index}] ==
assert [{third_transaction_hash, third_index}] ==
transaction.hash
|> Chain.transaction_to_internal_transactions(paging_options: %PagingOptions{key: {0}, page_size: 2})
|> Chain.transaction_to_internal_transactions(paging_options: %PagingOptions{key: {1}, page_size: 2})
|> Enum.map(&{&1.transaction_hash, &1.index})
end
end

@ -533,7 +533,8 @@ defmodule Explorer.EtherscanTest do
found_internal_transactions = Etherscan.list_internal_transactions(transaction.hash)
assert length(found_internal_transactions) == 3
# excluding of internal transactions with type=call and index=0
assert length(found_internal_transactions) == 2
end
test "only returns internal transactions that belong to the transaction" do
@ -571,7 +572,8 @@ defmodule Explorer.EtherscanTest do
internal_transactions1 = Etherscan.list_internal_transactions(transaction1.hash)
assert length(internal_transactions1) == 2
# excluding of internal transactions with type=call and index=0
assert length(internal_transactions1) == 1
internal_transactions2 = Etherscan.list_internal_transactions(transaction2.hash)
@ -659,7 +661,7 @@ defmodule Explorer.EtherscanTest do
|> insert()
|> with_block()
for index <- 0..2 do
for index <- 0..3 do
internal_transaction_details = %{
transaction: transaction,
index: index,
@ -719,7 +721,8 @@ defmodule Explorer.EtherscanTest do
internal_transactions1 = Etherscan.list_internal_transactions(address1.hash)
assert length(internal_transactions1) == 3
# excluding of internal transactions with type=call and index=0
assert length(internal_transactions1) == 2
internal_transactions2 = Etherscan.list_internal_transactions(address2.hash)
@ -734,7 +737,7 @@ defmodule Explorer.EtherscanTest do
|> insert()
|> with_block()
for index <- 0..2 do
for index <- 0..3 do
internal_transaction_details = %{
transaction: transaction,
index: index,
@ -795,7 +798,8 @@ defmodule Explorer.EtherscanTest do
expected_block_numbers = [second_block.number, third_block.number]
assert length(found_internal_transactions) == 4
# excluding of internal transactions with type=call and index=0
assert length(found_internal_transactions) == 2
for internal_transaction <- found_internal_transactions do
assert internal_transaction.block_number in expected_block_numbers

Loading…
Cancel
Save