From 1bd2dcdda21fe7c79097bfe5f05621a5230a162c Mon Sep 17 00:00:00 2001 From: pasqu4le Date: Sat, 25 May 2019 17:51:39 +0200 Subject: [PATCH 1/7] Fix BlocksTransactionsMismatch temporary fetcher This solves a problem that was found with this fetcher: block with no transactions were not taken into account, so they were not checked and never had their `refetch_needed` field set to false. --- CHANGELOG.md | 1 + .../indexer/temporary/blocks_transactions_mismatch.ex | 11 ++++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4edae41dad..546fe30432 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ - [#1966](https://github.com/poanetwork/blockscout/pull/1966) - fix: add fields for contract filter performance - [#2008](https://github.com/poanetwork/blockscout/pull/2008) - add new function clause for xDai network beneficiaries - [#2009](https://github.com/poanetwork/blockscout/pull/2009) - addresses page improvements +- [#2027](https://github.com/poanetwork/blockscout/pull/2027) - fix: `BlocksTransactionsMismatch` ignoring blocks without transactions ### Chore diff --git a/apps/indexer/lib/indexer/temporary/blocks_transactions_mismatch.ex b/apps/indexer/lib/indexer/temporary/blocks_transactions_mismatch.ex index 76f4322379..82dd0a2e55 100644 --- a/apps/indexer/lib/indexer/temporary/blocks_transactions_mismatch.ex +++ b/apps/indexer/lib/indexer/temporary/blocks_transactions_mismatch.ex @@ -51,7 +51,7 @@ defmodule Indexer.Temporary.BlocksTransactionsMismatch do def init(initial, reducer, _) do query = from(block in Block, - join: transactions in assoc(block, :transactions), + left_join: transactions in assoc(block, :transactions), where: block.consensus and block.refetch_needed, group_by: block.hash, select: {block, count(transactions.hash)} @@ -81,14 +81,19 @@ defmodule Indexer.Temporary.BlocksTransactionsMismatch do defp run_blocks(%Blocks{blocks_params: []}, blocks_data), do: {:retry, blocks_data} defp run_blocks( - %Blocks{transactions_params: transactions_params}, + %Blocks{transactions_params: transactions_params, blocks_params: blocks_params}, blocks_data ) do - found_blocks_map = + blocks_with_transactions_map = transactions_params |> Enum.group_by(&Map.fetch!(&1, :block_hash)) |> Map.new(fn {block_hash, trans_lst} -> {block_hash, Enum.count(trans_lst)} end) + found_blocks_map = + blocks_params + |> Map.new(&{Map.fetch!(&1, :hash), 0}) + |> Map.merge(blocks_with_transactions_map) + {found_blocks_data, missing_blocks_data} = Enum.split_with(blocks_data, fn {block, _trans_num} -> Map.has_key?(found_blocks_map, to_string(block.hash)) From 728d25bcf46f8ad75f846cf5e45b9a6266a919b9 Mon Sep 17 00:00:00 2001 From: pasqu4le Date: Mon, 27 May 2019 10:43:36 +0200 Subject: [PATCH 2/7] Add consensus filter to AddRefetchNeededToBlock migration It is unnecessary to set `refetch_needed` to blocks that do not yet have a consensus, so this small change can save some time in the migration and avoid some checking from the temporary fetcher. --- .../migrations/20190513134025_add_refetch_needed_to_block.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/explorer/priv/repo/migrations/20190513134025_add_refetch_needed_to_block.exs b/apps/explorer/priv/repo/migrations/20190513134025_add_refetch_needed_to_block.exs index 70ddac3e03..5380e2df14 100644 --- a/apps/explorer/priv/repo/migrations/20190513134025_add_refetch_needed_to_block.exs +++ b/apps/explorer/priv/repo/migrations/20190513134025_add_refetch_needed_to_block.exs @@ -6,6 +6,6 @@ defmodule Explorer.Repo.Migrations.AddRefetchNeededToBlock do add(:refetch_needed, :boolean, default: false) end - execute("UPDATE blocks SET refetch_needed = TRUE;", "") + execute("UPDATE blocks SET refetch_needed = TRUE WHERE consensus", "") end end From 817cc4a00fb1ac1e332142796544c121261eacf5 Mon Sep 17 00:00:00 2001 From: zachdaniel Date: Thu, 23 May 2019 11:18:45 -0400 Subject: [PATCH 3/7] fix: use better queries for listLogs endpoint --- CHANGELOG.md | 1 + .../api/rpc/logs_controller_test.exs | 10 +- apps/explorer/lib/explorer/etherscan/logs.ex | 118 +++++++++++++----- .../test/explorer/etherscan/logs_test.exs | 38 +++--- 4 files changed, 114 insertions(+), 53 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 043d57d31f..b718a527d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,7 @@ - [#2017](https://github.com/poanetwork/blockscout/pull/2017) - fix: fix to/from filters on tx list pages - [#2008](https://github.com/poanetwork/blockscout/pull/2008) - add new function clause for xDai network beneficiaries - [#2009](https://github.com/poanetwork/blockscout/pull/2009) - addresses page improvements +- [#2014](https://github.com/poanetwork/blockscout/pull/2014) - fix: use better queries for listLogs endpoint ### Chore diff --git a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/logs_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/logs_controller_test.exs index 758e37e801..454db88e86 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/logs_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/logs_controller_test.exs @@ -266,7 +266,7 @@ defmodule BlockScoutWeb.API.RPC.LogsControllerTest do transaction = %Transaction{block: block} = :transaction - |> insert() + |> insert(to_address: contract_address) |> with_block() log = insert(:log, address: contract_address, transaction: transaction) @@ -313,13 +313,13 @@ defmodule BlockScoutWeb.API.RPC.LogsControllerTest do transaction_block1 = %Transaction{} = :transaction - |> insert() + |> insert(to_address: contract_address) |> with_block(first_block) transaction_block2 = %Transaction{} = :transaction - |> insert() + |> insert(to_address: contract_address) |> with_block(second_block) insert(:log, address: contract_address, transaction: transaction_block1) @@ -356,13 +356,13 @@ defmodule BlockScoutWeb.API.RPC.LogsControllerTest do transaction_block1 = %Transaction{} = :transaction - |> insert() + |> insert(to_address: contract_address) |> with_block(first_block) transaction_block2 = %Transaction{} = :transaction - |> insert() + |> insert(to_address: contract_address) |> with_block(second_block) insert(:log, address: contract_address, transaction: transaction_block1) diff --git a/apps/explorer/lib/explorer/etherscan/logs.ex b/apps/explorer/lib/explorer/etherscan/logs.ex index e8cfb7404c..2228a748d6 100644 --- a/apps/explorer/lib/explorer/etherscan/logs.ex +++ b/apps/explorer/lib/explorer/etherscan/logs.ex @@ -5,9 +5,9 @@ defmodule Explorer.Etherscan.Logs do """ - import Ecto.Query, only: [from: 2, where: 3, subquery: 1, order_by: 3, limit: 2] + import Ecto.Query, only: [from: 2, where: 3, subquery: 1] - alias Explorer.Chain.Log + alias Explorer.Chain.{Block, InternalTransaction, Log, Transaction} alias Explorer.Repo @base_filter %{ @@ -67,33 +67,99 @@ defmodule Explorer.Etherscan.Logs do """ @spec list_logs(map()) :: [map()] - def list_logs(filter) do + def list_logs(%{address_hash: address_hash} = filter) when not is_nil(address_hash) do prepared_filter = Map.merge(@base_filter, filter) - query = - from( - l in Log, - inner_join: t in assoc(l, :transaction), - inner_join: b in assoc(t, :block), - where: b.number >= ^prepared_filter.from_block, - where: b.number <= ^prepared_filter.to_block, + logs_query = where_topic_match(Log, prepared_filter) + + internal_transaction_log_query = + from(internal_transaction in InternalTransaction, + join: transaction in assoc(internal_transaction, :transaction), + join: log in ^logs_query, + on: log.transaction_hash == internal_transaction.transaction_hash, + where: internal_transaction.block_number >= ^prepared_filter.from_block, + where: internal_transaction.block_number <= ^prepared_filter.to_block, + where: internal_transaction.to_address_hash == ^address_hash, + or_where: internal_transaction.from_address_hash == ^address_hash, + or_where: internal_transaction.created_contract_address_hash == ^address_hash, select: - merge(map(l, ^@log_fields), %{ - gas_price: t.gas_price, - gas_used: t.gas_used, - transaction_index: t.index, - block_number: b.number, - block_timestamp: b.timestamp + merge(map(log, ^@log_fields), %{ + gas_price: transaction.gas_price, + gas_used: transaction.gas_used, + transaction_index: transaction.index, + block_number: transaction.block_number }) ) - query - |> subquery() - |> where_address_match(prepared_filter) - |> where_topic_match(prepared_filter) - |> order_by([l], l.block_number) - |> limit(1_000) - |> Repo.all() + all_transaction_logs_query = + from(transaction in Transaction, + join: log in ^logs_query, + on: log.transaction_hash == transaction.hash, + where: transaction.block_number >= ^prepared_filter.from_block, + where: transaction.block_number <= ^prepared_filter.to_block, + where: transaction.to_address_hash == ^address_hash, + or_where: transaction.from_address_hash == ^address_hash, + or_where: transaction.created_contract_address_hash == ^address_hash, + select: + merge(map(log, ^@log_fields), %{ + gas_price: transaction.gas_price, + gas_used: transaction.gas_used, + transaction_index: transaction.index, + block_number: transaction.block_number + }), + union: ^internal_transaction_log_query + ) + + query_with_blocks = + from(log_transaction_data in subquery(all_transaction_logs_query), + join: block in Block, + on: block.number == log_transaction_data.block_number, + on: block.consensus == true, + where: log_transaction_data.address_hash == ^address_hash, + order_by: block.number, + limit: 1000, + select_merge: %{ + block_timestamp: block.timestamp + } + ) + + Repo.all(query_with_blocks) + end + + # Since address_hash was not present, we know that a + # topic filter has been applied, and th + def list_logs(filter) do + prepared_filter = Map.merge(@base_filter, filter) + + logs_query = where_topic_match(Log, prepared_filter) + + block_transaction_query = + from(transaction in Transaction, + join: block in assoc(transaction, :block), + on: block.number >= ^prepared_filter.from_block, + on: block.number <= ^prepared_filter.to_block, + on: block.consensus == true, + select: %{ + transaction_hash: transaction.hash, + gas_price: transaction.gas_price, + gas_used: transaction.gas_used, + transaction_index: transaction.index, + block_number: block.number, + block_timestamp: block.timestamp + } + ) + + query_with_block_transaction_data = + from(log in logs_query, + join: block_transaction_data in subquery(block_transaction_query), + on: block_transaction_data.transaction_hash == log.transaction_hash, + order_by: block_transaction_data.block_number, + limit: 1000, + select: block_transaction_data, + select_merge: map(log, ^@log_fields) + ) + + Repo.all(query_with_block_transaction_data) end @topics [ @@ -112,12 +178,6 @@ defmodule Explorer.Etherscan.Logs do topic2_3_opr: {:third_topic, :fourth_topic} } - defp where_address_match(query, %{address_hash: address_hash}) when not is_nil(address_hash) do - where(query, [l], l.address_hash == ^address_hash) - end - - defp where_address_match(query, _), do: query - defp where_topic_match(query, filter) do case Enum.filter(@topics, &filter[&1]) do [] -> diff --git a/apps/explorer/test/explorer/etherscan/logs_test.exs b/apps/explorer/test/explorer/etherscan/logs_test.exs index 65c62175f6..00050db6f4 100644 --- a/apps/explorer/test/explorer/etherscan/logs_test.exs +++ b/apps/explorer/test/explorer/etherscan/logs_test.exs @@ -24,7 +24,7 @@ defmodule Explorer.Etherscan.LogsTest do %Transaction{block: block} = :transaction - |> insert() + |> insert(to_address: contract_address) |> with_block() filter = %{ @@ -42,7 +42,7 @@ defmodule Explorer.Etherscan.LogsTest do transaction = %Transaction{block: block} = :transaction - |> insert() + |> insert(to_address: contract_address) |> with_block() log = insert(:log, address: contract_address, transaction: transaction) @@ -76,7 +76,7 @@ defmodule Explorer.Etherscan.LogsTest do transaction = %Transaction{block: block} = :transaction - |> insert() + |> insert(to_address: contract_address) |> with_block() insert_list(2, :log, address: contract_address, transaction: transaction) @@ -101,13 +101,13 @@ defmodule Explorer.Etherscan.LogsTest do transaction_block1 = %Transaction{} = :transaction - |> insert() + |> insert(to_address: contract_address) |> with_block(first_block) transaction_block2 = %Transaction{} = :transaction - |> insert() + |> insert(to_address: contract_address) |> with_block(second_block) insert(:log, address: contract_address, transaction: transaction_block1) @@ -134,13 +134,13 @@ defmodule Explorer.Etherscan.LogsTest do transaction_block1 = %Transaction{} = :transaction - |> insert() + |> insert(to_address: contract_address) |> with_block(first_block) transaction_block2 = %Transaction{} = :transaction - |> insert() + |> insert(to_address: contract_address) |> with_block(second_block) insert(:log, address: contract_address, transaction: transaction_block1) @@ -164,7 +164,7 @@ defmodule Explorer.Etherscan.LogsTest do transaction = %Transaction{block: block} = :transaction - |> insert() + |> insert(to_address: contract_address) |> with_block() log1_details = [ @@ -200,7 +200,7 @@ defmodule Explorer.Etherscan.LogsTest do transaction = %Transaction{block: block} = :transaction - |> insert() + |> insert(to_address: contract_address) |> with_block() log1_details = [ @@ -241,7 +241,7 @@ defmodule Explorer.Etherscan.LogsTest do transaction = %Transaction{block: block} = :transaction - |> insert() + |> insert(to_address: contract_address) |> with_block() log1_details = [ @@ -280,7 +280,7 @@ defmodule Explorer.Etherscan.LogsTest do transaction = %Transaction{block: block} = :transaction - |> insert() + |> insert(to_address: contract_address) |> with_block() log1_details = [ @@ -317,7 +317,7 @@ defmodule Explorer.Etherscan.LogsTest do transaction = %Transaction{block: block} = :transaction - |> insert() + |> insert(to_address: contract_address) |> with_block() log1_details = [ @@ -358,7 +358,7 @@ defmodule Explorer.Etherscan.LogsTest do transaction = %Transaction{block: block} = :transaction - |> insert() + |> insert(to_address: contract_address) |> with_block() log1_details = [ @@ -415,7 +415,7 @@ defmodule Explorer.Etherscan.LogsTest do transaction = %Transaction{block: block} = :transaction - |> insert() + |> insert(to_address: contract_address) |> with_block() log1_details = [ @@ -472,7 +472,7 @@ defmodule Explorer.Etherscan.LogsTest do transaction = %Transaction{block: block} = :transaction - |> insert() + |> insert(to_address: contract_address) |> with_block() log1_details = [ @@ -529,7 +529,7 @@ defmodule Explorer.Etherscan.LogsTest do transaction = %Transaction{block: block} = :transaction - |> insert() + |> insert(to_address: contract_address) |> with_block() log1_details = [ @@ -595,19 +595,19 @@ defmodule Explorer.Etherscan.LogsTest do transaction_block1 = %Transaction{} = :transaction - |> insert() + |> insert(to_address: contract_address) |> with_block(first_block) transaction_block2 = %Transaction{} = :transaction - |> insert() + |> insert(to_address: contract_address) |> with_block(second_block) transaction_block3 = %Transaction{} = :transaction - |> insert() + |> insert(to_address: contract_address) |> with_block(third_block) insert(:log, address: contract_address, transaction: transaction_block3) From 2ccdfab572a88e82c10d0e96a7211c8444069ecf Mon Sep 17 00:00:00 2001 From: zachdaniel Date: Thu, 23 May 2019 11:18:45 -0400 Subject: [PATCH 4/7] fix: use better queries for listLogs endpoint --- apps/explorer/lib/explorer/etherscan/logs.ex | 38 ++++++++++---------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/apps/explorer/lib/explorer/etherscan/logs.ex b/apps/explorer/lib/explorer/etherscan/logs.ex index 2228a748d6..124199be72 100644 --- a/apps/explorer/lib/explorer/etherscan/logs.ex +++ b/apps/explorer/lib/explorer/etherscan/logs.ex @@ -79,9 +79,10 @@ defmodule Explorer.Etherscan.Logs do on: log.transaction_hash == internal_transaction.transaction_hash, where: internal_transaction.block_number >= ^prepared_filter.from_block, where: internal_transaction.block_number <= ^prepared_filter.to_block, - where: internal_transaction.to_address_hash == ^address_hash, - or_where: internal_transaction.from_address_hash == ^address_hash, - or_where: internal_transaction.created_contract_address_hash == ^address_hash, + where: + internal_transaction.to_address_hash == ^address_hash or + internal_transaction.from_address_hash == ^address_hash or + internal_transaction.created_contract_address_hash == ^address_hash, select: merge(map(log, ^@log_fields), %{ gas_price: transaction.gas_price, @@ -97,16 +98,15 @@ defmodule Explorer.Etherscan.Logs do on: log.transaction_hash == transaction.hash, where: transaction.block_number >= ^prepared_filter.from_block, where: transaction.block_number <= ^prepared_filter.to_block, - where: transaction.to_address_hash == ^address_hash, - or_where: transaction.from_address_hash == ^address_hash, - or_where: transaction.created_contract_address_hash == ^address_hash, - select: - merge(map(log, ^@log_fields), %{ - gas_price: transaction.gas_price, - gas_used: transaction.gas_used, - transaction_index: transaction.index, - block_number: transaction.block_number - }), + where: + transaction.to_address_hash == ^address_hash or + transaction.from_address_hash == ^address_hash or + transaction.created_contract_address_hash == ^address_hash, + select: map(log, ^@log_fields), + select_merge: map(transaction, [:gas_price, :gas_used, :block_number]), + select_merge: %{ + transaction_index: transaction.index + }, union: ^internal_transaction_log_query ) @@ -114,7 +114,7 @@ defmodule Explorer.Etherscan.Logs do from(log_transaction_data in subquery(all_transaction_logs_query), join: block in Block, on: block.number == log_transaction_data.block_number, - on: block.consensus == true, + where: block.consensus == true, where: log_transaction_data.address_hash == ^address_hash, order_by: block.number, limit: 1000, @@ -127,7 +127,9 @@ defmodule Explorer.Etherscan.Logs do end # Since address_hash was not present, we know that a - # topic filter has been applied, and th + # topic filter has been applied, so we use a different + # query that is optimized for a logs filter over an + # address_hash def list_logs(filter) do prepared_filter = Map.merge(@base_filter, filter) @@ -136,9 +138,9 @@ defmodule Explorer.Etherscan.Logs do block_transaction_query = from(transaction in Transaction, join: block in assoc(transaction, :block), - on: block.number >= ^prepared_filter.from_block, - on: block.number <= ^prepared_filter.to_block, - on: block.consensus == true, + where: block.number >= ^prepared_filter.from_block, + where: block.number <= ^prepared_filter.to_block, + where: block.consensus == true, select: %{ transaction_hash: transaction.hash, gas_price: transaction.gas_price, From d01bc3ab3de8326521b7e682eee863fac1181f72 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Wed, 29 May 2019 11:31:30 +0300 Subject: [PATCH 5/7] add CHANGELOG entry for async pagination --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 043d57d31f..d95eb19b4f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,8 @@ - [#1999](https://github.com/poanetwork/blockscout/pull/1999) - load data async on addresses page - [#2002](https://github.com/poanetwork/blockscout/pull/2002) - Get estimated count of blocks when cache is empty - [#1807](https://github.com/poanetwork/blockscout/pull/1807) - New theming capabilites. -- [#2040](https://github.com/poanetwork/blockscout/pull/2040) - Verification links to other explorers for ETH +- [#2040](https://github.com/poanetwork/blockscout/pull/2040) - Verification links to other explorers for ETH +- [#2012](https://github.com/poanetwork/blockscout/pull/2012) - make all pages pagination async ### Fixes - [#2043](https://github.com/poanetwork/blockscout/pull/2043) - Fixed modal dialog width for 'verify other explorers' From 6a6a10ecb2ea196c6fdc7ff8dcd45b02706ee9b6 Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Wed, 29 May 2019 13:52:14 +0300 Subject: [PATCH 6/7] Increase timeout for geth indexers --- apps/indexer/config/prod/geth.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/indexer/config/prod/geth.exs b/apps/indexer/config/prod/geth.exs index cf1113119d..859d94319c 100644 --- a/apps/indexer/config/prod/geth.exs +++ b/apps/indexer/config/prod/geth.exs @@ -7,7 +7,7 @@ config :indexer, transport_options: [ http: EthereumJSONRPC.HTTP.HTTPoison, url: System.get_env("ETHEREUM_JSONRPC_HTTP_URL") || "https://mainnet.infura.io/8lTvJTKmHPCHazkneJsY", - http_options: [recv_timeout: :timer.minutes(1), timeout: :timer.minutes(1), hackney: [pool: :ethereum_jsonrpc]] + http_options: [recv_timeout: :timer.minutes(10), timeout: :timer.minutes(10), hackney: [pool: :ethereum_jsonrpc]] ], variant: EthereumJSONRPC.Geth ], From a0dce0578dc3dfac0abd5c414d001de9d5f57c93 Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Wed, 29 May 2019 13:55:24 +0300 Subject: [PATCH 7/7] Add CHANGELOG entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 043d57d31f..539349b975 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,7 @@ - [#1988](https://github.com/poanetwork/blockscout/pull/1988) - Fix wrong parity tasks names in Circle CI - [#2000](https://github.com/poanetwork/blockscout/pull/2000) - docker/Makefile: always set a container name - [#2018](https://github.com/poanetwork/blockscout/pull/2018) - Use PORT env variable in dev config +- [#2055](https://github.com/poanetwork/blockscout/pull/2055) - Increase timeout for geth indexers ## 1.3.14-beta