Merge branch 'master' into ab-do-not-load-assoc-for-trx-inter-trx-pagination

pull/2065/head
Ayrat Badykov 6 years ago committed by GitHub
commit 67b200d2c3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 6
      CHANGELOG.md
  2. 10
      apps/block_scout_web/test/block_scout_web/controllers/api/rpc/logs_controller_test.exs
  3. 120
      apps/explorer/lib/explorer/etherscan/logs.ex
  4. 2
      apps/explorer/priv/repo/migrations/20190513134025_add_refetch_needed_to_block.exs
  5. 38
      apps/explorer/test/explorer/etherscan/logs_test.exs
  6. 2
      apps/indexer/config/prod/geth.exs
  7. 11
      apps/indexer/lib/indexer/temporary/blocks_transactions_mismatch.ex

@ -18,7 +18,8 @@
- [#1999](https://github.com/poanetwork/blockscout/pull/1999) - load data async on addresses page - [#1999](https://github.com/poanetwork/blockscout/pull/1999) - load data async on addresses page
- [#2002](https://github.com/poanetwork/blockscout/pull/2002) - Get estimated count of blocks when cache is empty - [#2002](https://github.com/poanetwork/blockscout/pull/2002) - Get estimated count of blocks when cache is empty
- [#1807](https://github.com/poanetwork/blockscout/pull/1807) - New theming capabilites. - [#1807](https://github.com/poanetwork/blockscout/pull/1807) - New theming capabilites.
- [#2040](https://github.com/poanetwork/blockscout/pull/2040) - Verification links to other explorers for ETH - [#2040](https://github.com/poanetwork/blockscout/pull/2040) - Verification links to other explorers for ETH
- [#2012](https://github.com/poanetwork/blockscout/pull/2012) - make all pages pagination async
### Fixes ### Fixes
- [#2043](https://github.com/poanetwork/blockscout/pull/2043) - Fixed modal dialog width for 'verify other explorers' - [#2043](https://github.com/poanetwork/blockscout/pull/2043) - Fixed modal dialog width for 'verify other explorers'
@ -44,6 +45,8 @@
- [#2017](https://github.com/poanetwork/blockscout/pull/2017) - fix: fix to/from filters on tx list pages - [#2017](https://github.com/poanetwork/blockscout/pull/2017) - fix: fix to/from filters on tx list pages
- [#2008](https://github.com/poanetwork/blockscout/pull/2008) - add new function clause for xDai network beneficiaries - [#2008](https://github.com/poanetwork/blockscout/pull/2008) - add new function clause for xDai network beneficiaries
- [#2009](https://github.com/poanetwork/blockscout/pull/2009) - addresses page improvements - [#2009](https://github.com/poanetwork/blockscout/pull/2009) - addresses page improvements
- [#2014](https://github.com/poanetwork/blockscout/pull/2014) - fix: use better queries for listLogs endpoint
- [#2027](https://github.com/poanetwork/blockscout/pull/2027) - fix: `BlocksTransactionsMismatch` ignoring blocks without transactions
### Chore ### Chore
@ -54,6 +57,7 @@
- [#1988](https://github.com/poanetwork/blockscout/pull/1988) - Fix wrong parity tasks names in Circle CI - [#1988](https://github.com/poanetwork/blockscout/pull/1988) - Fix wrong parity tasks names in Circle CI
- [#2000](https://github.com/poanetwork/blockscout/pull/2000) - docker/Makefile: always set a container name - [#2000](https://github.com/poanetwork/blockscout/pull/2000) - docker/Makefile: always set a container name
- [#2018](https://github.com/poanetwork/blockscout/pull/2018) - Use PORT env variable in dev config - [#2018](https://github.com/poanetwork/blockscout/pull/2018) - Use PORT env variable in dev config
- [#2055](https://github.com/poanetwork/blockscout/pull/2055) - Increase timeout for geth indexers
## 1.3.14-beta ## 1.3.14-beta

@ -266,7 +266,7 @@ defmodule BlockScoutWeb.API.RPC.LogsControllerTest do
transaction = transaction =
%Transaction{block: block} = %Transaction{block: block} =
:transaction :transaction
|> insert() |> insert(to_address: contract_address)
|> with_block() |> with_block()
log = insert(:log, address: contract_address, transaction: transaction) log = insert(:log, address: contract_address, transaction: transaction)
@ -313,13 +313,13 @@ defmodule BlockScoutWeb.API.RPC.LogsControllerTest do
transaction_block1 = transaction_block1 =
%Transaction{} = %Transaction{} =
:transaction :transaction
|> insert() |> insert(to_address: contract_address)
|> with_block(first_block) |> with_block(first_block)
transaction_block2 = transaction_block2 =
%Transaction{} = %Transaction{} =
:transaction :transaction
|> insert() |> insert(to_address: contract_address)
|> with_block(second_block) |> with_block(second_block)
insert(:log, address: contract_address, transaction: transaction_block1) insert(:log, address: contract_address, transaction: transaction_block1)
@ -356,13 +356,13 @@ defmodule BlockScoutWeb.API.RPC.LogsControllerTest do
transaction_block1 = transaction_block1 =
%Transaction{} = %Transaction{} =
:transaction :transaction
|> insert() |> insert(to_address: contract_address)
|> with_block(first_block) |> with_block(first_block)
transaction_block2 = transaction_block2 =
%Transaction{} = %Transaction{} =
:transaction :transaction
|> insert() |> insert(to_address: contract_address)
|> with_block(second_block) |> with_block(second_block)
insert(:log, address: contract_address, transaction: transaction_block1) insert(:log, address: contract_address, transaction: transaction_block1)

@ -5,9 +5,9 @@ defmodule Explorer.Etherscan.Logs do
""" """
import Ecto.Query, only: [from: 2, where: 3, subquery: 1, order_by: 3, limit: 2] import Ecto.Query, only: [from: 2, where: 3, subquery: 1]
alias Explorer.Chain.Log alias Explorer.Chain.{Block, InternalTransaction, Log, Transaction}
alias Explorer.Repo alias Explorer.Repo
@base_filter %{ @base_filter %{
@ -67,33 +67,101 @@ defmodule Explorer.Etherscan.Logs do
""" """
@spec list_logs(map()) :: [map()] @spec list_logs(map()) :: [map()]
def list_logs(filter) do def list_logs(%{address_hash: address_hash} = filter) when not is_nil(address_hash) do
prepared_filter = Map.merge(@base_filter, filter) prepared_filter = Map.merge(@base_filter, filter)
query = logs_query = where_topic_match(Log, prepared_filter)
from(
l in Log, internal_transaction_log_query =
inner_join: t in assoc(l, :transaction), from(internal_transaction in InternalTransaction,
inner_join: b in assoc(t, :block), join: transaction in assoc(internal_transaction, :transaction),
where: b.number >= ^prepared_filter.from_block, join: log in ^logs_query,
where: b.number <= ^prepared_filter.to_block, on: log.transaction_hash == internal_transaction.transaction_hash,
where: internal_transaction.block_number >= ^prepared_filter.from_block,
where: internal_transaction.block_number <= ^prepared_filter.to_block,
where:
internal_transaction.to_address_hash == ^address_hash or
internal_transaction.from_address_hash == ^address_hash or
internal_transaction.created_contract_address_hash == ^address_hash,
select: select:
merge(map(l, ^@log_fields), %{ merge(map(log, ^@log_fields), %{
gas_price: t.gas_price, gas_price: transaction.gas_price,
gas_used: t.gas_used, gas_used: transaction.gas_used,
transaction_index: t.index, transaction_index: transaction.index,
block_number: b.number, block_number: transaction.block_number
block_timestamp: b.timestamp
}) })
) )
query all_transaction_logs_query =
|> subquery() from(transaction in Transaction,
|> where_address_match(prepared_filter) join: log in ^logs_query,
|> where_topic_match(prepared_filter) on: log.transaction_hash == transaction.hash,
|> order_by([l], l.block_number) where: transaction.block_number >= ^prepared_filter.from_block,
|> limit(1_000) where: transaction.block_number <= ^prepared_filter.to_block,
|> Repo.all() where:
transaction.to_address_hash == ^address_hash or
transaction.from_address_hash == ^address_hash or
transaction.created_contract_address_hash == ^address_hash,
select: map(log, ^@log_fields),
select_merge: map(transaction, [:gas_price, :gas_used, :block_number]),
select_merge: %{
transaction_index: transaction.index
},
union: ^internal_transaction_log_query
)
query_with_blocks =
from(log_transaction_data in subquery(all_transaction_logs_query),
join: block in Block,
on: block.number == log_transaction_data.block_number,
where: block.consensus == true,
where: log_transaction_data.address_hash == ^address_hash,
order_by: block.number,
limit: 1000,
select_merge: %{
block_timestamp: block.timestamp
}
)
Repo.all(query_with_blocks)
end
# Since address_hash was not present, we know that a
# topic filter has been applied, so we use a different
# query that is optimized for a logs filter over an
# address_hash
def list_logs(filter) do
prepared_filter = Map.merge(@base_filter, filter)
logs_query = where_topic_match(Log, prepared_filter)
block_transaction_query =
from(transaction in Transaction,
join: block in assoc(transaction, :block),
where: block.number >= ^prepared_filter.from_block,
where: block.number <= ^prepared_filter.to_block,
where: block.consensus == true,
select: %{
transaction_hash: transaction.hash,
gas_price: transaction.gas_price,
gas_used: transaction.gas_used,
transaction_index: transaction.index,
block_number: block.number,
block_timestamp: block.timestamp
}
)
query_with_block_transaction_data =
from(log in logs_query,
join: block_transaction_data in subquery(block_transaction_query),
on: block_transaction_data.transaction_hash == log.transaction_hash,
order_by: block_transaction_data.block_number,
limit: 1000,
select: block_transaction_data,
select_merge: map(log, ^@log_fields)
)
Repo.all(query_with_block_transaction_data)
end end
@topics [ @topics [
@ -112,12 +180,6 @@ defmodule Explorer.Etherscan.Logs do
topic2_3_opr: {:third_topic, :fourth_topic} topic2_3_opr: {:third_topic, :fourth_topic}
} }
defp where_address_match(query, %{address_hash: address_hash}) when not is_nil(address_hash) do
where(query, [l], l.address_hash == ^address_hash)
end
defp where_address_match(query, _), do: query
defp where_topic_match(query, filter) do defp where_topic_match(query, filter) do
case Enum.filter(@topics, &filter[&1]) do case Enum.filter(@topics, &filter[&1]) do
[] -> [] ->

@ -6,6 +6,6 @@ defmodule Explorer.Repo.Migrations.AddRefetchNeededToBlock do
add(:refetch_needed, :boolean, default: false) add(:refetch_needed, :boolean, default: false)
end end
execute("UPDATE blocks SET refetch_needed = TRUE;", "") execute("UPDATE blocks SET refetch_needed = TRUE WHERE consensus", "")
end end
end end

@ -24,7 +24,7 @@ defmodule Explorer.Etherscan.LogsTest do
%Transaction{block: block} = %Transaction{block: block} =
:transaction :transaction
|> insert() |> insert(to_address: contract_address)
|> with_block() |> with_block()
filter = %{ filter = %{
@ -42,7 +42,7 @@ defmodule Explorer.Etherscan.LogsTest do
transaction = transaction =
%Transaction{block: block} = %Transaction{block: block} =
:transaction :transaction
|> insert() |> insert(to_address: contract_address)
|> with_block() |> with_block()
log = insert(:log, address: contract_address, transaction: transaction) log = insert(:log, address: contract_address, transaction: transaction)
@ -76,7 +76,7 @@ defmodule Explorer.Etherscan.LogsTest do
transaction = transaction =
%Transaction{block: block} = %Transaction{block: block} =
:transaction :transaction
|> insert() |> insert(to_address: contract_address)
|> with_block() |> with_block()
insert_list(2, :log, address: contract_address, transaction: transaction) insert_list(2, :log, address: contract_address, transaction: transaction)
@ -101,13 +101,13 @@ defmodule Explorer.Etherscan.LogsTest do
transaction_block1 = transaction_block1 =
%Transaction{} = %Transaction{} =
:transaction :transaction
|> insert() |> insert(to_address: contract_address)
|> with_block(first_block) |> with_block(first_block)
transaction_block2 = transaction_block2 =
%Transaction{} = %Transaction{} =
:transaction :transaction
|> insert() |> insert(to_address: contract_address)
|> with_block(second_block) |> with_block(second_block)
insert(:log, address: contract_address, transaction: transaction_block1) insert(:log, address: contract_address, transaction: transaction_block1)
@ -134,13 +134,13 @@ defmodule Explorer.Etherscan.LogsTest do
transaction_block1 = transaction_block1 =
%Transaction{} = %Transaction{} =
:transaction :transaction
|> insert() |> insert(to_address: contract_address)
|> with_block(first_block) |> with_block(first_block)
transaction_block2 = transaction_block2 =
%Transaction{} = %Transaction{} =
:transaction :transaction
|> insert() |> insert(to_address: contract_address)
|> with_block(second_block) |> with_block(second_block)
insert(:log, address: contract_address, transaction: transaction_block1) insert(:log, address: contract_address, transaction: transaction_block1)
@ -164,7 +164,7 @@ defmodule Explorer.Etherscan.LogsTest do
transaction = transaction =
%Transaction{block: block} = %Transaction{block: block} =
:transaction :transaction
|> insert() |> insert(to_address: contract_address)
|> with_block() |> with_block()
log1_details = [ log1_details = [
@ -200,7 +200,7 @@ defmodule Explorer.Etherscan.LogsTest do
transaction = transaction =
%Transaction{block: block} = %Transaction{block: block} =
:transaction :transaction
|> insert() |> insert(to_address: contract_address)
|> with_block() |> with_block()
log1_details = [ log1_details = [
@ -241,7 +241,7 @@ defmodule Explorer.Etherscan.LogsTest do
transaction = transaction =
%Transaction{block: block} = %Transaction{block: block} =
:transaction :transaction
|> insert() |> insert(to_address: contract_address)
|> with_block() |> with_block()
log1_details = [ log1_details = [
@ -280,7 +280,7 @@ defmodule Explorer.Etherscan.LogsTest do
transaction = transaction =
%Transaction{block: block} = %Transaction{block: block} =
:transaction :transaction
|> insert() |> insert(to_address: contract_address)
|> with_block() |> with_block()
log1_details = [ log1_details = [
@ -317,7 +317,7 @@ defmodule Explorer.Etherscan.LogsTest do
transaction = transaction =
%Transaction{block: block} = %Transaction{block: block} =
:transaction :transaction
|> insert() |> insert(to_address: contract_address)
|> with_block() |> with_block()
log1_details = [ log1_details = [
@ -358,7 +358,7 @@ defmodule Explorer.Etherscan.LogsTest do
transaction = transaction =
%Transaction{block: block} = %Transaction{block: block} =
:transaction :transaction
|> insert() |> insert(to_address: contract_address)
|> with_block() |> with_block()
log1_details = [ log1_details = [
@ -415,7 +415,7 @@ defmodule Explorer.Etherscan.LogsTest do
transaction = transaction =
%Transaction{block: block} = %Transaction{block: block} =
:transaction :transaction
|> insert() |> insert(to_address: contract_address)
|> with_block() |> with_block()
log1_details = [ log1_details = [
@ -472,7 +472,7 @@ defmodule Explorer.Etherscan.LogsTest do
transaction = transaction =
%Transaction{block: block} = %Transaction{block: block} =
:transaction :transaction
|> insert() |> insert(to_address: contract_address)
|> with_block() |> with_block()
log1_details = [ log1_details = [
@ -529,7 +529,7 @@ defmodule Explorer.Etherscan.LogsTest do
transaction = transaction =
%Transaction{block: block} = %Transaction{block: block} =
:transaction :transaction
|> insert() |> insert(to_address: contract_address)
|> with_block() |> with_block()
log1_details = [ log1_details = [
@ -595,19 +595,19 @@ defmodule Explorer.Etherscan.LogsTest do
transaction_block1 = transaction_block1 =
%Transaction{} = %Transaction{} =
:transaction :transaction
|> insert() |> insert(to_address: contract_address)
|> with_block(first_block) |> with_block(first_block)
transaction_block2 = transaction_block2 =
%Transaction{} = %Transaction{} =
:transaction :transaction
|> insert() |> insert(to_address: contract_address)
|> with_block(second_block) |> with_block(second_block)
transaction_block3 = transaction_block3 =
%Transaction{} = %Transaction{} =
:transaction :transaction
|> insert() |> insert(to_address: contract_address)
|> with_block(third_block) |> with_block(third_block)
insert(:log, address: contract_address, transaction: transaction_block3) insert(:log, address: contract_address, transaction: transaction_block3)

@ -7,7 +7,7 @@ config :indexer,
transport_options: [ transport_options: [
http: EthereumJSONRPC.HTTP.HTTPoison, http: EthereumJSONRPC.HTTP.HTTPoison,
url: System.get_env("ETHEREUM_JSONRPC_HTTP_URL") || "https://mainnet.infura.io/8lTvJTKmHPCHazkneJsY", url: System.get_env("ETHEREUM_JSONRPC_HTTP_URL") || "https://mainnet.infura.io/8lTvJTKmHPCHazkneJsY",
http_options: [recv_timeout: :timer.minutes(1), timeout: :timer.minutes(1), hackney: [pool: :ethereum_jsonrpc]] http_options: [recv_timeout: :timer.minutes(10), timeout: :timer.minutes(10), hackney: [pool: :ethereum_jsonrpc]]
], ],
variant: EthereumJSONRPC.Geth variant: EthereumJSONRPC.Geth
], ],

@ -51,7 +51,7 @@ defmodule Indexer.Temporary.BlocksTransactionsMismatch do
def init(initial, reducer, _) do def init(initial, reducer, _) do
query = query =
from(block in Block, from(block in Block,
join: transactions in assoc(block, :transactions), left_join: transactions in assoc(block, :transactions),
where: block.consensus and block.refetch_needed, where: block.consensus and block.refetch_needed,
group_by: block.hash, group_by: block.hash,
select: {block, count(transactions.hash)} select: {block, count(transactions.hash)}
@ -81,14 +81,19 @@ defmodule Indexer.Temporary.BlocksTransactionsMismatch do
defp run_blocks(%Blocks{blocks_params: []}, blocks_data), do: {:retry, blocks_data} defp run_blocks(%Blocks{blocks_params: []}, blocks_data), do: {:retry, blocks_data}
defp run_blocks( defp run_blocks(
%Blocks{transactions_params: transactions_params}, %Blocks{transactions_params: transactions_params, blocks_params: blocks_params},
blocks_data blocks_data
) do ) do
found_blocks_map = blocks_with_transactions_map =
transactions_params transactions_params
|> Enum.group_by(&Map.fetch!(&1, :block_hash)) |> Enum.group_by(&Map.fetch!(&1, :block_hash))
|> Map.new(fn {block_hash, trans_lst} -> {block_hash, Enum.count(trans_lst)} end) |> Map.new(fn {block_hash, trans_lst} -> {block_hash, Enum.count(trans_lst)} end)
found_blocks_map =
blocks_params
|> Map.new(&{Map.fetch!(&1, :hash), 0})
|> Map.merge(blocks_with_transactions_map)
{found_blocks_data, missing_blocks_data} = {found_blocks_data, missing_blocks_data} =
Enum.split_with(blocks_data, fn {block, _trans_num} -> Enum.split_with(blocks_data, fn {block, _trans_num} ->
Map.has_key?(found_blocks_map, to_string(block.hash)) Map.has_key?(found_blocks_map, to_string(block.hash))

Loading…
Cancel
Save