fix: use better queries for listLogs endpoint

pull/2014/head
zachdaniel 6 years ago
parent 608485ff25
commit 817cc4a00f
  1. 1
      CHANGELOG.md
  2. 10
      apps/block_scout_web/test/block_scout_web/controllers/api/rpc/logs_controller_test.exs
  3. 118
      apps/explorer/lib/explorer/etherscan/logs.ex
  4. 38
      apps/explorer/test/explorer/etherscan/logs_test.exs

@ -44,6 +44,7 @@
- [#2017](https://github.com/poanetwork/blockscout/pull/2017) - fix: fix to/from filters on tx list pages - [#2017](https://github.com/poanetwork/blockscout/pull/2017) - fix: fix to/from filters on tx list pages
- [#2008](https://github.com/poanetwork/blockscout/pull/2008) - add new function clause for xDai network beneficiaries - [#2008](https://github.com/poanetwork/blockscout/pull/2008) - add new function clause for xDai network beneficiaries
- [#2009](https://github.com/poanetwork/blockscout/pull/2009) - addresses page improvements - [#2009](https://github.com/poanetwork/blockscout/pull/2009) - addresses page improvements
- [#2014](https://github.com/poanetwork/blockscout/pull/2014) - fix: use better queries for listLogs endpoint
### Chore ### Chore

@ -266,7 +266,7 @@ defmodule BlockScoutWeb.API.RPC.LogsControllerTest do
transaction = transaction =
%Transaction{block: block} = %Transaction{block: block} =
:transaction :transaction
|> insert() |> insert(to_address: contract_address)
|> with_block() |> with_block()
log = insert(:log, address: contract_address, transaction: transaction) log = insert(:log, address: contract_address, transaction: transaction)
@ -313,13 +313,13 @@ defmodule BlockScoutWeb.API.RPC.LogsControllerTest do
transaction_block1 = transaction_block1 =
%Transaction{} = %Transaction{} =
:transaction :transaction
|> insert() |> insert(to_address: contract_address)
|> with_block(first_block) |> with_block(first_block)
transaction_block2 = transaction_block2 =
%Transaction{} = %Transaction{} =
:transaction :transaction
|> insert() |> insert(to_address: contract_address)
|> with_block(second_block) |> with_block(second_block)
insert(:log, address: contract_address, transaction: transaction_block1) insert(:log, address: contract_address, transaction: transaction_block1)
@ -356,13 +356,13 @@ defmodule BlockScoutWeb.API.RPC.LogsControllerTest do
transaction_block1 = transaction_block1 =
%Transaction{} = %Transaction{} =
:transaction :transaction
|> insert() |> insert(to_address: contract_address)
|> with_block(first_block) |> with_block(first_block)
transaction_block2 = transaction_block2 =
%Transaction{} = %Transaction{} =
:transaction :transaction
|> insert() |> insert(to_address: contract_address)
|> with_block(second_block) |> with_block(second_block)
insert(:log, address: contract_address, transaction: transaction_block1) insert(:log, address: contract_address, transaction: transaction_block1)

@ -5,9 +5,9 @@ defmodule Explorer.Etherscan.Logs do
""" """
import Ecto.Query, only: [from: 2, where: 3, subquery: 1, order_by: 3, limit: 2] import Ecto.Query, only: [from: 2, where: 3, subquery: 1]
alias Explorer.Chain.Log alias Explorer.Chain.{Block, InternalTransaction, Log, Transaction}
alias Explorer.Repo alias Explorer.Repo
@base_filter %{ @base_filter %{
@ -67,33 +67,99 @@ defmodule Explorer.Etherscan.Logs do
""" """
@spec list_logs(map()) :: [map()] @spec list_logs(map()) :: [map()]
def list_logs(filter) do def list_logs(%{address_hash: address_hash} = filter) when not is_nil(address_hash) do
prepared_filter = Map.merge(@base_filter, filter) prepared_filter = Map.merge(@base_filter, filter)
query = logs_query = where_topic_match(Log, prepared_filter)
from(
l in Log, internal_transaction_log_query =
inner_join: t in assoc(l, :transaction), from(internal_transaction in InternalTransaction,
inner_join: b in assoc(t, :block), join: transaction in assoc(internal_transaction, :transaction),
where: b.number >= ^prepared_filter.from_block, join: log in ^logs_query,
where: b.number <= ^prepared_filter.to_block, on: log.transaction_hash == internal_transaction.transaction_hash,
where: internal_transaction.block_number >= ^prepared_filter.from_block,
where: internal_transaction.block_number <= ^prepared_filter.to_block,
where: internal_transaction.to_address_hash == ^address_hash,
or_where: internal_transaction.from_address_hash == ^address_hash,
or_where: internal_transaction.created_contract_address_hash == ^address_hash,
select: select:
merge(map(l, ^@log_fields), %{ merge(map(log, ^@log_fields), %{
gas_price: t.gas_price, gas_price: transaction.gas_price,
gas_used: t.gas_used, gas_used: transaction.gas_used,
transaction_index: t.index, transaction_index: transaction.index,
block_number: b.number, block_number: transaction.block_number
block_timestamp: b.timestamp
}) })
) )
query all_transaction_logs_query =
|> subquery() from(transaction in Transaction,
|> where_address_match(prepared_filter) join: log in ^logs_query,
|> where_topic_match(prepared_filter) on: log.transaction_hash == transaction.hash,
|> order_by([l], l.block_number) where: transaction.block_number >= ^prepared_filter.from_block,
|> limit(1_000) where: transaction.block_number <= ^prepared_filter.to_block,
|> Repo.all() where: transaction.to_address_hash == ^address_hash,
or_where: transaction.from_address_hash == ^address_hash,
or_where: transaction.created_contract_address_hash == ^address_hash,
select:
merge(map(log, ^@log_fields), %{
gas_price: transaction.gas_price,
gas_used: transaction.gas_used,
transaction_index: transaction.index,
block_number: transaction.block_number
}),
union: ^internal_transaction_log_query
)
query_with_blocks =
from(log_transaction_data in subquery(all_transaction_logs_query),
join: block in Block,
on: block.number == log_transaction_data.block_number,
on: block.consensus == true,
where: log_transaction_data.address_hash == ^address_hash,
order_by: block.number,
limit: 1000,
select_merge: %{
block_timestamp: block.timestamp
}
)
Repo.all(query_with_blocks)
end
# Since address_hash was not present, we know that a
# topic filter has been applied, and th
def list_logs(filter) do
prepared_filter = Map.merge(@base_filter, filter)
logs_query = where_topic_match(Log, prepared_filter)
block_transaction_query =
from(transaction in Transaction,
join: block in assoc(transaction, :block),
on: block.number >= ^prepared_filter.from_block,
on: block.number <= ^prepared_filter.to_block,
on: block.consensus == true,
select: %{
transaction_hash: transaction.hash,
gas_price: transaction.gas_price,
gas_used: transaction.gas_used,
transaction_index: transaction.index,
block_number: block.number,
block_timestamp: block.timestamp
}
)
query_with_block_transaction_data =
from(log in logs_query,
join: block_transaction_data in subquery(block_transaction_query),
on: block_transaction_data.transaction_hash == log.transaction_hash,
order_by: block_transaction_data.block_number,
limit: 1000,
select: block_transaction_data,
select_merge: map(log, ^@log_fields)
)
Repo.all(query_with_block_transaction_data)
end end
@topics [ @topics [
@ -112,12 +178,6 @@ defmodule Explorer.Etherscan.Logs do
topic2_3_opr: {:third_topic, :fourth_topic} topic2_3_opr: {:third_topic, :fourth_topic}
} }
defp where_address_match(query, %{address_hash: address_hash}) when not is_nil(address_hash) do
where(query, [l], l.address_hash == ^address_hash)
end
defp where_address_match(query, _), do: query
defp where_topic_match(query, filter) do defp where_topic_match(query, filter) do
case Enum.filter(@topics, &filter[&1]) do case Enum.filter(@topics, &filter[&1]) do
[] -> [] ->

@ -24,7 +24,7 @@ defmodule Explorer.Etherscan.LogsTest do
%Transaction{block: block} = %Transaction{block: block} =
:transaction :transaction
|> insert() |> insert(to_address: contract_address)
|> with_block() |> with_block()
filter = %{ filter = %{
@ -42,7 +42,7 @@ defmodule Explorer.Etherscan.LogsTest do
transaction = transaction =
%Transaction{block: block} = %Transaction{block: block} =
:transaction :transaction
|> insert() |> insert(to_address: contract_address)
|> with_block() |> with_block()
log = insert(:log, address: contract_address, transaction: transaction) log = insert(:log, address: contract_address, transaction: transaction)
@ -76,7 +76,7 @@ defmodule Explorer.Etherscan.LogsTest do
transaction = transaction =
%Transaction{block: block} = %Transaction{block: block} =
:transaction :transaction
|> insert() |> insert(to_address: contract_address)
|> with_block() |> with_block()
insert_list(2, :log, address: contract_address, transaction: transaction) insert_list(2, :log, address: contract_address, transaction: transaction)
@ -101,13 +101,13 @@ defmodule Explorer.Etherscan.LogsTest do
transaction_block1 = transaction_block1 =
%Transaction{} = %Transaction{} =
:transaction :transaction
|> insert() |> insert(to_address: contract_address)
|> with_block(first_block) |> with_block(first_block)
transaction_block2 = transaction_block2 =
%Transaction{} = %Transaction{} =
:transaction :transaction
|> insert() |> insert(to_address: contract_address)
|> with_block(second_block) |> with_block(second_block)
insert(:log, address: contract_address, transaction: transaction_block1) insert(:log, address: contract_address, transaction: transaction_block1)
@ -134,13 +134,13 @@ defmodule Explorer.Etherscan.LogsTest do
transaction_block1 = transaction_block1 =
%Transaction{} = %Transaction{} =
:transaction :transaction
|> insert() |> insert(to_address: contract_address)
|> with_block(first_block) |> with_block(first_block)
transaction_block2 = transaction_block2 =
%Transaction{} = %Transaction{} =
:transaction :transaction
|> insert() |> insert(to_address: contract_address)
|> with_block(second_block) |> with_block(second_block)
insert(:log, address: contract_address, transaction: transaction_block1) insert(:log, address: contract_address, transaction: transaction_block1)
@ -164,7 +164,7 @@ defmodule Explorer.Etherscan.LogsTest do
transaction = transaction =
%Transaction{block: block} = %Transaction{block: block} =
:transaction :transaction
|> insert() |> insert(to_address: contract_address)
|> with_block() |> with_block()
log1_details = [ log1_details = [
@ -200,7 +200,7 @@ defmodule Explorer.Etherscan.LogsTest do
transaction = transaction =
%Transaction{block: block} = %Transaction{block: block} =
:transaction :transaction
|> insert() |> insert(to_address: contract_address)
|> with_block() |> with_block()
log1_details = [ log1_details = [
@ -241,7 +241,7 @@ defmodule Explorer.Etherscan.LogsTest do
transaction = transaction =
%Transaction{block: block} = %Transaction{block: block} =
:transaction :transaction
|> insert() |> insert(to_address: contract_address)
|> with_block() |> with_block()
log1_details = [ log1_details = [
@ -280,7 +280,7 @@ defmodule Explorer.Etherscan.LogsTest do
transaction = transaction =
%Transaction{block: block} = %Transaction{block: block} =
:transaction :transaction
|> insert() |> insert(to_address: contract_address)
|> with_block() |> with_block()
log1_details = [ log1_details = [
@ -317,7 +317,7 @@ defmodule Explorer.Etherscan.LogsTest do
transaction = transaction =
%Transaction{block: block} = %Transaction{block: block} =
:transaction :transaction
|> insert() |> insert(to_address: contract_address)
|> with_block() |> with_block()
log1_details = [ log1_details = [
@ -358,7 +358,7 @@ defmodule Explorer.Etherscan.LogsTest do
transaction = transaction =
%Transaction{block: block} = %Transaction{block: block} =
:transaction :transaction
|> insert() |> insert(to_address: contract_address)
|> with_block() |> with_block()
log1_details = [ log1_details = [
@ -415,7 +415,7 @@ defmodule Explorer.Etherscan.LogsTest do
transaction = transaction =
%Transaction{block: block} = %Transaction{block: block} =
:transaction :transaction
|> insert() |> insert(to_address: contract_address)
|> with_block() |> with_block()
log1_details = [ log1_details = [
@ -472,7 +472,7 @@ defmodule Explorer.Etherscan.LogsTest do
transaction = transaction =
%Transaction{block: block} = %Transaction{block: block} =
:transaction :transaction
|> insert() |> insert(to_address: contract_address)
|> with_block() |> with_block()
log1_details = [ log1_details = [
@ -529,7 +529,7 @@ defmodule Explorer.Etherscan.LogsTest do
transaction = transaction =
%Transaction{block: block} = %Transaction{block: block} =
:transaction :transaction
|> insert() |> insert(to_address: contract_address)
|> with_block() |> with_block()
log1_details = [ log1_details = [
@ -595,19 +595,19 @@ defmodule Explorer.Etherscan.LogsTest do
transaction_block1 = transaction_block1 =
%Transaction{} = %Transaction{} =
:transaction :transaction
|> insert() |> insert(to_address: contract_address)
|> with_block(first_block) |> with_block(first_block)
transaction_block2 = transaction_block2 =
%Transaction{} = %Transaction{} =
:transaction :transaction
|> insert() |> insert(to_address: contract_address)
|> with_block(second_block) |> with_block(second_block)
transaction_block3 = transaction_block3 =
%Transaction{} = %Transaction{} =
:transaction :transaction
|> insert() |> insert(to_address: contract_address)
|> with_block(third_block) |> with_block(third_block)
insert(:log, address: contract_address, transaction: transaction_block3) insert(:log, address: contract_address, transaction: transaction_block3)

Loading…
Cancel
Save