diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e290fff1d..eb59cc0e50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,8 @@ - [#1999](https://github.com/poanetwork/blockscout/pull/1999) - load data async on addresses page - [#2002](https://github.com/poanetwork/blockscout/pull/2002) - Get estimated count of blocks when cache is empty - [#1807](https://github.com/poanetwork/blockscout/pull/1807) - New theming capabilites. -- [#2040](https://github.com/poanetwork/blockscout/pull/2040) - Verification links to other explorers for ETH +- [#2040](https://github.com/poanetwork/blockscout/pull/2040) - Verification links to other explorers for ETH +- [#2012](https://github.com/poanetwork/blockscout/pull/2012) - make all pages pagination async ### Fixes - [#2043](https://github.com/poanetwork/blockscout/pull/2043) - Fixed modal dialog width for 'verify other explorers' @@ -45,6 +46,8 @@ - [#2008](https://github.com/poanetwork/blockscout/pull/2008) - add new function clause for xDai network beneficiaries - [#2009](https://github.com/poanetwork/blockscout/pull/2009) - addresses page improvements - [#2047](https://github.com/poanetwork/blockscout/pull/2047) - fix: show creating internal transactions +- [#2014](https://github.com/poanetwork/blockscout/pull/2014) - fix: use better queries for listLogs endpoint +- [#2027](https://github.com/poanetwork/blockscout/pull/2027) - fix: `BlocksTransactionsMismatch` ignoring blocks without transactions ### Chore @@ -55,6 +58,7 @@ - [#1988](https://github.com/poanetwork/blockscout/pull/1988) - Fix wrong parity tasks names in Circle CI - [#2000](https://github.com/poanetwork/blockscout/pull/2000) - docker/Makefile: always set a container name - [#2018](https://github.com/poanetwork/blockscout/pull/2018) - Use PORT env variable in dev config +- [#2055](https://github.com/poanetwork/blockscout/pull/2055) - Increase timeout for geth indexers ## 1.3.14-beta diff --git a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/logs_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/logs_controller_test.exs index 758e37e801..454db88e86 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/logs_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/logs_controller_test.exs @@ -266,7 +266,7 @@ defmodule BlockScoutWeb.API.RPC.LogsControllerTest do transaction = %Transaction{block: block} = :transaction - |> insert() + |> insert(to_address: contract_address) |> with_block() log = insert(:log, address: contract_address, transaction: transaction) @@ -313,13 +313,13 @@ defmodule BlockScoutWeb.API.RPC.LogsControllerTest do transaction_block1 = %Transaction{} = :transaction - |> insert() + |> insert(to_address: contract_address) |> with_block(first_block) transaction_block2 = %Transaction{} = :transaction - |> insert() + |> insert(to_address: contract_address) |> with_block(second_block) insert(:log, address: contract_address, transaction: transaction_block1) @@ -356,13 +356,13 @@ defmodule BlockScoutWeb.API.RPC.LogsControllerTest do transaction_block1 = %Transaction{} = :transaction - |> insert() + |> insert(to_address: contract_address) |> with_block(first_block) transaction_block2 = %Transaction{} = :transaction - |> insert() + |> insert(to_address: contract_address) |> with_block(second_block) insert(:log, address: contract_address, transaction: transaction_block1) diff --git a/apps/explorer/lib/explorer/etherscan/logs.ex b/apps/explorer/lib/explorer/etherscan/logs.ex index e8cfb7404c..124199be72 100644 --- a/apps/explorer/lib/explorer/etherscan/logs.ex +++ b/apps/explorer/lib/explorer/etherscan/logs.ex @@ -5,9 +5,9 @@ defmodule Explorer.Etherscan.Logs do """ - import Ecto.Query, only: [from: 2, where: 3, subquery: 1, order_by: 3, limit: 2] + import Ecto.Query, only: [from: 2, where: 3, subquery: 1] - alias Explorer.Chain.Log + alias Explorer.Chain.{Block, InternalTransaction, Log, Transaction} alias Explorer.Repo @base_filter %{ @@ -67,33 +67,101 @@ defmodule Explorer.Etherscan.Logs do """ @spec list_logs(map()) :: [map()] - def list_logs(filter) do + def list_logs(%{address_hash: address_hash} = filter) when not is_nil(address_hash) do prepared_filter = Map.merge(@base_filter, filter) - query = - from( - l in Log, - inner_join: t in assoc(l, :transaction), - inner_join: b in assoc(t, :block), - where: b.number >= ^prepared_filter.from_block, - where: b.number <= ^prepared_filter.to_block, + logs_query = where_topic_match(Log, prepared_filter) + + internal_transaction_log_query = + from(internal_transaction in InternalTransaction, + join: transaction in assoc(internal_transaction, :transaction), + join: log in ^logs_query, + on: log.transaction_hash == internal_transaction.transaction_hash, + where: internal_transaction.block_number >= ^prepared_filter.from_block, + where: internal_transaction.block_number <= ^prepared_filter.to_block, + where: + internal_transaction.to_address_hash == ^address_hash or + internal_transaction.from_address_hash == ^address_hash or + internal_transaction.created_contract_address_hash == ^address_hash, select: - merge(map(l, ^@log_fields), %{ - gas_price: t.gas_price, - gas_used: t.gas_used, - transaction_index: t.index, - block_number: b.number, - block_timestamp: b.timestamp + merge(map(log, ^@log_fields), %{ + gas_price: transaction.gas_price, + gas_used: transaction.gas_used, + transaction_index: transaction.index, + block_number: transaction.block_number }) ) - query - |> subquery() - |> where_address_match(prepared_filter) - |> where_topic_match(prepared_filter) - |> order_by([l], l.block_number) - |> limit(1_000) - |> Repo.all() + all_transaction_logs_query = + from(transaction in Transaction, + join: log in ^logs_query, + on: log.transaction_hash == transaction.hash, + where: transaction.block_number >= ^prepared_filter.from_block, + where: transaction.block_number <= ^prepared_filter.to_block, + where: + transaction.to_address_hash == ^address_hash or + transaction.from_address_hash == ^address_hash or + transaction.created_contract_address_hash == ^address_hash, + select: map(log, ^@log_fields), + select_merge: map(transaction, [:gas_price, :gas_used, :block_number]), + select_merge: %{ + transaction_index: transaction.index + }, + union: ^internal_transaction_log_query + ) + + query_with_blocks = + from(log_transaction_data in subquery(all_transaction_logs_query), + join: block in Block, + on: block.number == log_transaction_data.block_number, + where: block.consensus == true, + where: log_transaction_data.address_hash == ^address_hash, + order_by: block.number, + limit: 1000, + select_merge: %{ + block_timestamp: block.timestamp + } + ) + + Repo.all(query_with_blocks) + end + + # Since address_hash was not present, we know that a + # topic filter has been applied, so we use a different + # query that is optimized for a logs filter over an + # address_hash + def list_logs(filter) do + prepared_filter = Map.merge(@base_filter, filter) + + logs_query = where_topic_match(Log, prepared_filter) + + block_transaction_query = + from(transaction in Transaction, + join: block in assoc(transaction, :block), + where: block.number >= ^prepared_filter.from_block, + where: block.number <= ^prepared_filter.to_block, + where: block.consensus == true, + select: %{ + transaction_hash: transaction.hash, + gas_price: transaction.gas_price, + gas_used: transaction.gas_used, + transaction_index: transaction.index, + block_number: block.number, + block_timestamp: block.timestamp + } + ) + + query_with_block_transaction_data = + from(log in logs_query, + join: block_transaction_data in subquery(block_transaction_query), + on: block_transaction_data.transaction_hash == log.transaction_hash, + order_by: block_transaction_data.block_number, + limit: 1000, + select: block_transaction_data, + select_merge: map(log, ^@log_fields) + ) + + Repo.all(query_with_block_transaction_data) end @topics [ @@ -112,12 +180,6 @@ defmodule Explorer.Etherscan.Logs do topic2_3_opr: {:third_topic, :fourth_topic} } - defp where_address_match(query, %{address_hash: address_hash}) when not is_nil(address_hash) do - where(query, [l], l.address_hash == ^address_hash) - end - - defp where_address_match(query, _), do: query - defp where_topic_match(query, filter) do case Enum.filter(@topics, &filter[&1]) do [] -> diff --git a/apps/explorer/priv/repo/migrations/20190513134025_add_refetch_needed_to_block.exs b/apps/explorer/priv/repo/migrations/20190513134025_add_refetch_needed_to_block.exs index 70ddac3e03..5380e2df14 100644 --- a/apps/explorer/priv/repo/migrations/20190513134025_add_refetch_needed_to_block.exs +++ b/apps/explorer/priv/repo/migrations/20190513134025_add_refetch_needed_to_block.exs @@ -6,6 +6,6 @@ defmodule Explorer.Repo.Migrations.AddRefetchNeededToBlock do add(:refetch_needed, :boolean, default: false) end - execute("UPDATE blocks SET refetch_needed = TRUE;", "") + execute("UPDATE blocks SET refetch_needed = TRUE WHERE consensus", "") end end diff --git a/apps/explorer/test/explorer/etherscan/logs_test.exs b/apps/explorer/test/explorer/etherscan/logs_test.exs index 65c62175f6..00050db6f4 100644 --- a/apps/explorer/test/explorer/etherscan/logs_test.exs +++ b/apps/explorer/test/explorer/etherscan/logs_test.exs @@ -24,7 +24,7 @@ defmodule Explorer.Etherscan.LogsTest do %Transaction{block: block} = :transaction - |> insert() + |> insert(to_address: contract_address) |> with_block() filter = %{ @@ -42,7 +42,7 @@ defmodule Explorer.Etherscan.LogsTest do transaction = %Transaction{block: block} = :transaction - |> insert() + |> insert(to_address: contract_address) |> with_block() log = insert(:log, address: contract_address, transaction: transaction) @@ -76,7 +76,7 @@ defmodule Explorer.Etherscan.LogsTest do transaction = %Transaction{block: block} = :transaction - |> insert() + |> insert(to_address: contract_address) |> with_block() insert_list(2, :log, address: contract_address, transaction: transaction) @@ -101,13 +101,13 @@ defmodule Explorer.Etherscan.LogsTest do transaction_block1 = %Transaction{} = :transaction - |> insert() + |> insert(to_address: contract_address) |> with_block(first_block) transaction_block2 = %Transaction{} = :transaction - |> insert() + |> insert(to_address: contract_address) |> with_block(second_block) insert(:log, address: contract_address, transaction: transaction_block1) @@ -134,13 +134,13 @@ defmodule Explorer.Etherscan.LogsTest do transaction_block1 = %Transaction{} = :transaction - |> insert() + |> insert(to_address: contract_address) |> with_block(first_block) transaction_block2 = %Transaction{} = :transaction - |> insert() + |> insert(to_address: contract_address) |> with_block(second_block) insert(:log, address: contract_address, transaction: transaction_block1) @@ -164,7 +164,7 @@ defmodule Explorer.Etherscan.LogsTest do transaction = %Transaction{block: block} = :transaction - |> insert() + |> insert(to_address: contract_address) |> with_block() log1_details = [ @@ -200,7 +200,7 @@ defmodule Explorer.Etherscan.LogsTest do transaction = %Transaction{block: block} = :transaction - |> insert() + |> insert(to_address: contract_address) |> with_block() log1_details = [ @@ -241,7 +241,7 @@ defmodule Explorer.Etherscan.LogsTest do transaction = %Transaction{block: block} = :transaction - |> insert() + |> insert(to_address: contract_address) |> with_block() log1_details = [ @@ -280,7 +280,7 @@ defmodule Explorer.Etherscan.LogsTest do transaction = %Transaction{block: block} = :transaction - |> insert() + |> insert(to_address: contract_address) |> with_block() log1_details = [ @@ -317,7 +317,7 @@ defmodule Explorer.Etherscan.LogsTest do transaction = %Transaction{block: block} = :transaction - |> insert() + |> insert(to_address: contract_address) |> with_block() log1_details = [ @@ -358,7 +358,7 @@ defmodule Explorer.Etherscan.LogsTest do transaction = %Transaction{block: block} = :transaction - |> insert() + |> insert(to_address: contract_address) |> with_block() log1_details = [ @@ -415,7 +415,7 @@ defmodule Explorer.Etherscan.LogsTest do transaction = %Transaction{block: block} = :transaction - |> insert() + |> insert(to_address: contract_address) |> with_block() log1_details = [ @@ -472,7 +472,7 @@ defmodule Explorer.Etherscan.LogsTest do transaction = %Transaction{block: block} = :transaction - |> insert() + |> insert(to_address: contract_address) |> with_block() log1_details = [ @@ -529,7 +529,7 @@ defmodule Explorer.Etherscan.LogsTest do transaction = %Transaction{block: block} = :transaction - |> insert() + |> insert(to_address: contract_address) |> with_block() log1_details = [ @@ -595,19 +595,19 @@ defmodule Explorer.Etherscan.LogsTest do transaction_block1 = %Transaction{} = :transaction - |> insert() + |> insert(to_address: contract_address) |> with_block(first_block) transaction_block2 = %Transaction{} = :transaction - |> insert() + |> insert(to_address: contract_address) |> with_block(second_block) transaction_block3 = %Transaction{} = :transaction - |> insert() + |> insert(to_address: contract_address) |> with_block(third_block) insert(:log, address: contract_address, transaction: transaction_block3) diff --git a/apps/indexer/config/prod/geth.exs b/apps/indexer/config/prod/geth.exs index cf1113119d..859d94319c 100644 --- a/apps/indexer/config/prod/geth.exs +++ b/apps/indexer/config/prod/geth.exs @@ -7,7 +7,7 @@ config :indexer, transport_options: [ http: EthereumJSONRPC.HTTP.HTTPoison, url: System.get_env("ETHEREUM_JSONRPC_HTTP_URL") || "https://mainnet.infura.io/8lTvJTKmHPCHazkneJsY", - http_options: [recv_timeout: :timer.minutes(1), timeout: :timer.minutes(1), hackney: [pool: :ethereum_jsonrpc]] + http_options: [recv_timeout: :timer.minutes(10), timeout: :timer.minutes(10), hackney: [pool: :ethereum_jsonrpc]] ], variant: EthereumJSONRPC.Geth ], diff --git a/apps/indexer/lib/indexer/temporary/blocks_transactions_mismatch.ex b/apps/indexer/lib/indexer/temporary/blocks_transactions_mismatch.ex index 76f4322379..82dd0a2e55 100644 --- a/apps/indexer/lib/indexer/temporary/blocks_transactions_mismatch.ex +++ b/apps/indexer/lib/indexer/temporary/blocks_transactions_mismatch.ex @@ -51,7 +51,7 @@ defmodule Indexer.Temporary.BlocksTransactionsMismatch do def init(initial, reducer, _) do query = from(block in Block, - join: transactions in assoc(block, :transactions), + left_join: transactions in assoc(block, :transactions), where: block.consensus and block.refetch_needed, group_by: block.hash, select: {block, count(transactions.hash)} @@ -81,14 +81,19 @@ defmodule Indexer.Temporary.BlocksTransactionsMismatch do defp run_blocks(%Blocks{blocks_params: []}, blocks_data), do: {:retry, blocks_data} defp run_blocks( - %Blocks{transactions_params: transactions_params}, + %Blocks{transactions_params: transactions_params, blocks_params: blocks_params}, blocks_data ) do - found_blocks_map = + blocks_with_transactions_map = transactions_params |> Enum.group_by(&Map.fetch!(&1, :block_hash)) |> Map.new(fn {block_hash, trans_lst} -> {block_hash, Enum.count(trans_lst)} end) + found_blocks_map = + blocks_params + |> Map.new(&{Map.fetch!(&1, :hash), 0}) + |> Map.merge(blocks_with_transactions_map) + {found_blocks_data, missing_blocks_data} = Enum.split_with(blocks_data, fn {block, _trans_num} -> Map.has_key?(found_blocks_map, to_string(block.hash))