From fec9509027a575449e56c3a1c9091d2e7badebec Mon Sep 17 00:00:00 2001 From: Viktor Baranov Date: Wed, 13 Dec 2023 13:19:11 +0300 Subject: [PATCH 01/33] Refine token transfers token ids index --- CHANGELOG.md | 10 +++++++ ...20231213085254_add_btree_gin_extension.exs | 11 ++++++++ ...token_contract_address_token_ids_index.exs | 26 +++++++++++++++++++ ...5_drop_token_transfers_token_ids_index.exs | 7 +++++ 4 files changed, 54 insertions(+) create mode 100644 apps/explorer/priv/repo/migrations/20231213085254_add_btree_gin_extension.exs create mode 100644 apps/explorer/priv/repo/migrations/20231213090140_add_token_transfers_token_contract_address_token_ids_index.exs create mode 100644 apps/explorer/priv/repo/migrations/20231213101235_drop_token_transfers_token_ids_index.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index e08716581c..556786c55a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # ChangeLog +## 6.0.0-dev + +### Features + +### Fixes + +### Chore + +- [#8996](https://github.com/blockscout/blockscout/pull/8996) - Refine token transfers token ids index + ## Current ### Features diff --git a/apps/explorer/priv/repo/migrations/20231213085254_add_btree_gin_extension.exs b/apps/explorer/priv/repo/migrations/20231213085254_add_btree_gin_extension.exs new file mode 100644 index 0000000000..b34f9ee059 --- /dev/null +++ b/apps/explorer/priv/repo/migrations/20231213085254_add_btree_gin_extension.exs @@ -0,0 +1,11 @@ +defmodule Explorer.Repo.Migrations.CreateBtreeGinExtension do + use Ecto.Migration + + def up do + execute("CREATE EXTENSION IF NOT EXISTS btree_gin") + end + + def down do + execute("DROP EXTENSION IF EXISTS btree_gin") + end +end diff --git a/apps/explorer/priv/repo/migrations/20231213090140_add_token_transfers_token_contract_address_token_ids_index.exs b/apps/explorer/priv/repo/migrations/20231213090140_add_token_transfers_token_contract_address_token_ids_index.exs new file mode 100644 index 0000000000..7636fc7b3c --- /dev/null +++ b/apps/explorer/priv/repo/migrations/20231213090140_add_token_transfers_token_contract_address_token_ids_index.exs @@ -0,0 +1,26 @@ +defmodule Explorer.Repo.Migrations.AddTokenTransfersTokenContractAddressTokenIdsIndex do + use Ecto.Migration + @disable_ddl_transaction true + @disable_migration_lock true + + def up do + create( + index( + :token_transfers, + [:token_contract_address_hash, :token_ids], + name: "token_transfers_token_contract_address_hash_token_ids_index", + using: "GIN", + concurrently: true + ) + ) + end + + def down do + drop_if_exists( + index(:token_transfers, [:token_contract_address_hash, :token_ids], + name: :token_transfers_token_contract_address_hash_token_ids_index + ), + concurrently: true + ) + end +end diff --git a/apps/explorer/priv/repo/migrations/20231213101235_drop_token_transfers_token_ids_index.exs b/apps/explorer/priv/repo/migrations/20231213101235_drop_token_transfers_token_ids_index.exs new file mode 100644 index 0000000000..0065d2e263 --- /dev/null +++ b/apps/explorer/priv/repo/migrations/20231213101235_drop_token_transfers_token_ids_index.exs @@ -0,0 +1,7 @@ +defmodule Explorer.Repo.Migrations.DropTokenTransfersTokenIdsIndex do + use Ecto.Migration + + def change do + drop_if_exists(index(:token_transfers, [:token_ids], name: :token_transfers_token_ids_index)) + end +end From 6cbbe1f2ae0c7f37d237cb6ac52d0ef67b8e4f70 Mon Sep 17 00:00:00 2001 From: Viktor Baranov Date: Fri, 15 Dec 2023 13:50:10 +0300 Subject: [PATCH 02/33] Drop unused indexes on address_current_token_balances table --- CHANGELOG.md | 2 +- ...0231215104320_drop_unused_actb_indexes.exs | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 apps/explorer/priv/repo/migrations/20231215104320_drop_unused_actb_indexes.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index 556786c55a..98b074f2e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ ### Chore +- [#9006](https://github.com/blockscout/blockscout/pull/9006) - Drop unused indexes on address_current_token_balances table - [#8996](https://github.com/blockscout/blockscout/pull/8996) - Refine token transfers token ids index ## Current @@ -57,7 +58,6 @@ - [#9094](https://github.com/blockscout/blockscout/pull/9094) - Improve exchange rates logging - [#9014](https://github.com/blockscout/blockscout/pull/9014) - Decrease amount of NFT in address collection: 15 -> 9 - [#8994](https://github.com/blockscout/blockscout/pull/8994) - Refactor transactions event preloads -- [#8991](https://github.com/blockscout/blockscout/pull/8991) - Manage DB queue target via runtime env var
Dependencies version bumps diff --git a/apps/explorer/priv/repo/migrations/20231215104320_drop_unused_actb_indexes.exs b/apps/explorer/priv/repo/migrations/20231215104320_drop_unused_actb_indexes.exs new file mode 100644 index 0000000000..0f8ad39f3b --- /dev/null +++ b/apps/explorer/priv/repo/migrations/20231215104320_drop_unused_actb_indexes.exs @@ -0,0 +1,21 @@ +defmodule Explorer.Repo.Migrations.DropUnusedActbIndexes do + use Ecto.Migration + + def change do + drop( + index( + :address_current_token_balances, + [:value], + name: :address_current_token_balances_value, + where: "value IS NOT NULL" + ) + ) + + drop( + index( + :address_current_token_balances, + [:address_hash, :block_number, :token_contract_address_hash] + ) + ) + end +end From a176c38825ce0bbd3dfdb52ea7b60a326926aa3f Mon Sep 17 00:00:00 2001 From: Viktor Baranov Date: Fri, 15 Dec 2023 17:56:42 +0300 Subject: [PATCH 03/33] Index for block refetch_needed --- CHANGELOG.md | 1 + ...1215132609_add_index_blocks_refetch_needed.exs | 15 +++++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 apps/explorer/priv/repo/migrations/20231215132609_add_index_blocks_refetch_needed.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index 98b074f2e3..f267dd91bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,6 +57,7 @@ - [#9094](https://github.com/blockscout/blockscout/pull/9094) - Improve exchange rates logging - [#9014](https://github.com/blockscout/blockscout/pull/9014) - Decrease amount of NFT in address collection: 15 -> 9 +- [#9009](https://github.com/blockscout/blockscout/pull/9009) - Index for block refetch_needed - [#8994](https://github.com/blockscout/blockscout/pull/8994) - Refactor transactions event preloads
diff --git a/apps/explorer/priv/repo/migrations/20231215132609_add_index_blocks_refetch_needed.exs b/apps/explorer/priv/repo/migrations/20231215132609_add_index_blocks_refetch_needed.exs new file mode 100644 index 0000000000..f9a0c25ef6 --- /dev/null +++ b/apps/explorer/priv/repo/migrations/20231215132609_add_index_blocks_refetch_needed.exs @@ -0,0 +1,15 @@ +defmodule Explorer.Repo.Migrations.AddIndexBlocksRefetchNeeded do + use Ecto.Migration + + def up do + execute(""" + CREATE INDEX consensus_block_hashes_refetch_needed ON blocks((1)) WHERE consensus and refetch_needed; + """) + end + + def down do + execute(""" + DROP INDEX consensus_block_hashes_refetch_needed; + """) + end +end From f6f225d772e6b8a7702ae8f334ad0a2a831fc748 Mon Sep 17 00:00:00 2001 From: Viktor Baranov Date: Wed, 20 Dec 2023 15:50:59 +0300 Subject: [PATCH 04/33] Change index: base it to hash column --- .../20231215132609_add_index_blocks_refetch_needed.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/explorer/priv/repo/migrations/20231215132609_add_index_blocks_refetch_needed.exs b/apps/explorer/priv/repo/migrations/20231215132609_add_index_blocks_refetch_needed.exs index f9a0c25ef6..aef72e5ed8 100644 --- a/apps/explorer/priv/repo/migrations/20231215132609_add_index_blocks_refetch_needed.exs +++ b/apps/explorer/priv/repo/migrations/20231215132609_add_index_blocks_refetch_needed.exs @@ -3,7 +3,7 @@ defmodule Explorer.Repo.Migrations.AddIndexBlocksRefetchNeeded do def up do execute(""" - CREATE INDEX consensus_block_hashes_refetch_needed ON blocks((1)) WHERE consensus and refetch_needed; + CREATE INDEX consensus_block_hashes_refetch_needed ON blocks(hash) WHERE consensus and refetch_needed; """) end From 1d6c25ddb850efd2ad96a85763339579f114ea58 Mon Sep 17 00:00:00 2001 From: Viktor Baranov Date: Thu, 21 Dec 2023 14:52:05 +0300 Subject: [PATCH 05/33] Update CHANGELOG entry --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f267dd91bf..9f8e0db54f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ ### Chore +- [#9009](https://github.com/blockscout/blockscout/pull/9009) - Index for block refetch_needed - [#9006](https://github.com/blockscout/blockscout/pull/9006) - Drop unused indexes on address_current_token_balances table - [#8996](https://github.com/blockscout/blockscout/pull/8996) - Refine token transfers token ids index @@ -57,7 +58,6 @@ - [#9094](https://github.com/blockscout/blockscout/pull/9094) - Improve exchange rates logging - [#9014](https://github.com/blockscout/blockscout/pull/9014) - Decrease amount of NFT in address collection: 15 -> 9 -- [#9009](https://github.com/blockscout/blockscout/pull/9009) - Index for block refetch_needed - [#8994](https://github.com/blockscout/blockscout/pull/8994) - Refactor transactions event preloads
From 9a42fc88d0e53cec43247cee6736190949322d09 Mon Sep 17 00:00:00 2001 From: Viktor Baranov Date: Tue, 15 Mar 2022 13:02:46 +0300 Subject: [PATCH 06/33] Block consensus and timestamp in transaction table --- .../api/rpc/address_controller_test.exs | 6 +- .../lib/ethereum_jsonrpc/blocks.ex | 34 ++++++++- apps/explorer/lib/explorer/chain.ex | 13 ++-- ...dress_internal_transaction_csv_exporter.ex | 0 .../address_token_transfer_csv_exporter.ex | 0 .../chain/address_transaction_csv_exporter.ex | 0 .../explorer/chain/import/runner/blocks.ex | 13 ++++ .../import/runner/internal_transactions.ex | 27 +++++++- .../chain/import/runner/transactions.ex | 47 ++++++++++++- .../lib/explorer/chain/transaction.ex | 10 ++- .../chain/transaction/history/historian.ex | 23 ++----- apps/explorer/lib/explorer/etherscan.ex | 69 +++++++++++-------- apps/explorer/lib/explorer/etherscan/logs.ex | 45 ++++++------ ...902_add_consensus_to_transaction_table.exs | 19 +++++ ...d_block_timestamp_to_transaction_table.exs | 18 +++++ ...dress_token_transfer_csv_exporter_test.exs | 2 +- apps/explorer/test/explorer/chain_test.exs | 30 ++++---- .../test/explorer/etherscan/logs_test.exs | 7 +- .../explorer/test/explorer/etherscan_test.exs | 17 +++-- apps/explorer/test/support/factory.ex | 3 +- .../indexer/block/catchup/fetcher_test.exs | 2 +- .../fetcher/internal_transaction_test.exs | 2 +- .../test/indexer/fetcher/uncle_block_test.exs | 2 +- 23 files changed, 272 insertions(+), 117 deletions(-) create mode 100644 apps/explorer/lib/explorer/chain/address_internal_transaction_csv_exporter.ex create mode 100644 apps/explorer/lib/explorer/chain/address_token_transfer_csv_exporter.ex create mode 100644 apps/explorer/lib/explorer/chain/address_transaction_csv_exporter.ex create mode 100644 apps/explorer/priv/repo/migrations/20220315082902_add_consensus_to_transaction_table.exs create mode 100644 apps/explorer/priv/repo/migrations/20220315093927_add_block_timestamp_to_transaction_table.exs diff --git a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/address_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/address_controller_test.exs index ae0b2f3b94..f872c8928f 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/address_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/address_controller_test.exs @@ -1246,7 +1246,7 @@ defmodule BlockScoutWeb.API.RPC.AddressControllerTest do for block <- Enum.concat([blocks1, blocks2, blocks3]) do 2 - |> insert_list(:transaction, from_address: address) + |> insert_list(:transaction, from_address: address, block_timestamp: block.timestamp) |> with_block(block) end @@ -1294,7 +1294,7 @@ defmodule BlockScoutWeb.API.RPC.AddressControllerTest do for block <- Enum.concat([blocks1, blocks2, blocks3]) do 2 - |> insert_list(:transaction, from_address: address) + |> insert_list(:transaction, from_address: address, block_timestamp: block.timestamp) |> with_block(block) end @@ -1342,7 +1342,7 @@ defmodule BlockScoutWeb.API.RPC.AddressControllerTest do for block <- Enum.concat([blocks1, blocks2, blocks3]) do 2 - |> insert_list(:transaction, from_address: address) + |> insert_list(:transaction, from_address: address, block_timestamp: block.timestamp) |> with_block(block) end diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/blocks.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/blocks.ex index d9a697c1ac..a504468f49 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/blocks.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/blocks.ex @@ -54,12 +54,13 @@ defmodule EthereumJSONRPC.Blocks do transactions_params = Transactions.elixir_to_params(elixir_transactions) withdrawals_params = Withdrawals.elixir_to_params(elixir_withdrawals) blocks_params = elixir_to_params(elixir_blocks) + transactions_params_with_block_timestamp = add_timestamp_to_transactions_params(transactions_params, blocks_params) %__MODULE__{ errors: errors, blocks_params: blocks_params, block_second_degree_relations_params: block_second_degree_relations_params, - transactions_params: transactions_params, + transactions_params: transactions_params_with_block_timestamp, withdrawals_params: withdrawals_params } end @@ -454,4 +455,35 @@ defmodule EthereumJSONRPC.Blocks do def to_elixir(blocks) when is_list(blocks) do Enum.map(blocks, &Block.to_elixir/1) end + + defp add_timestamp_to_transactions_params(transactions_params, blocks_params) do + block_hashes = + transactions_params + |> Enum.map(fn %{block_hash: block_hash} -> block_hash end) + |> Enum.uniq() + + block_hash_timestamp_map = + block_hashes + |> Enum.map(fn block_hash -> + block = + Enum.find(blocks_params, fn block_param -> + block_param.hash == block_hash + end) + + %{} + |> Map.put("#{block_hash}", block.timestamp) + end) + |> Enum.reduce(%{}, fn hash_timestamp_map_item, acc -> + Map.merge(acc, hash_timestamp_map_item) + end) + + transactions_params + |> Enum.map(fn transactions_param -> + Map.put( + transactions_param, + :block_timestamp, + Map.get(block_hash_timestamp_map, "#{transactions_param.block_hash}") + ) + end) + end end diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index 2b7e30635b..af2d6874a6 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -522,15 +522,18 @@ defmodule Explorer.Chain do def gas_payment_by_block_hash(block_hashes) when is_list(block_hashes) do query = from( - block in Block, - left_join: transaction in assoc(block, :transactions), - where: block.hash in ^block_hashes and block.consensus == true, - group_by: block.hash, - select: {block.hash, %Wei{value: coalesce(sum(transaction.gas_used * transaction.gas_price), 0)}} + transaction in Transaction, + where: transaction.block_hash in ^block_hashes and transaction.block_consensus == true, + group_by: transaction.block_hash, + select: {transaction.block_hash, %Wei{value: coalesce(sum(transaction.gas_used * transaction.gas_price), 0)}} ) query |> Repo.all() + |> (&if(Enum.count(&1) > 0, + do: &1, + else: Enum.zip([block_hashes, for(_ <- 1..Enum.count(block_hashes), do: %Wei{value: Decimal.new(0)})]) + )).() |> Enum.into(%{}) end diff --git a/apps/explorer/lib/explorer/chain/address_internal_transaction_csv_exporter.ex b/apps/explorer/lib/explorer/chain/address_internal_transaction_csv_exporter.ex new file mode 100644 index 0000000000..e69de29bb2 diff --git a/apps/explorer/lib/explorer/chain/address_token_transfer_csv_exporter.ex b/apps/explorer/lib/explorer/chain/address_token_transfer_csv_exporter.ex new file mode 100644 index 0000000000..e69de29bb2 diff --git a/apps/explorer/lib/explorer/chain/address_transaction_csv_exporter.ex b/apps/explorer/lib/explorer/chain/address_transaction_csv_exporter.ex new file mode 100644 index 0000000000..e69de29bb2 diff --git a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex index 83a426b313..ffb8c47d5c 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex @@ -395,6 +395,19 @@ defmodule Explorer.Chain.Import.Runner.Blocks do timeout: timeout ) + repo.update_all( + from( + transaction in Transaction, + join: s in subquery(acquire_query), + on: transaction.block_hash == s.hash, + # we don't want to remove consensus from blocks that will be upserted + where: transaction.block_hash not in ^hashes, + select: transaction.block_hash + ), + [set: [block_consensus: false, updated_at: updated_at]], + timeout: timeout + ) + removed_consensus_block_hashes |> Enum.map(fn {number, _hash} -> number end) |> Enum.reject(&Enum.member?(consensus_block_numbers, &1)) diff --git a/apps/explorer/lib/explorer/chain/import/runner/internal_transactions.ex b/apps/explorer/lib/explorer/chain/import/runner/internal_transactions.ex index 2ef9cfe1fe..a4d512e602 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/internal_transactions.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/internal_transactions.ex @@ -693,7 +693,7 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactions do defp remove_consensus_of_invalid_blocks(repo, invalid_block_numbers) do if Enum.count(invalid_block_numbers) > 0 do - update_query = + update_block_query = from( block in Block, where: block.number in ^invalid_block_numbers and block.consensus == true, @@ -703,8 +703,18 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactions do update: [set: [consensus: false]] ) + update_transaction_query = + from( + transaction in Transaction, + where: transaction.block_number in ^invalid_block_numbers and transaction.block_consensus, + where: ^traceable_transactions_dynamic_query(), + # ShareLocks order already enforced by `acquire_blocks` (see docs: sharelocks.md) + update: [set: [block_consensus: false]] + ) + try do - {_num, result} = repo.update_all(update_query, []) + {_num, result} = repo.update_all(update_block_query, []) + {_num, _result} = repo.update_all(update_transaction_query, []) MissingRangesManipulator.add_ranges_by_block_numbers(invalid_block_numbers) @@ -762,4 +772,17 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactions do dynamic([_], true) end end + + defp traceable_transactions_dynamic_query do + if RangesHelper.trace_ranges_present?() do + block_ranges = RangesHelper.get_trace_block_ranges() + + Enum.reduce(block_ranges, dynamic([_], false), fn + _from.._to = range, acc -> dynamic([transaction], ^acc or transaction.block_number in ^range) + num_to_latest, acc -> dynamic([transaction], ^acc or transaction.block_number >= ^num_to_latest) + end) + else + dynamic([_], true) + end + end end diff --git a/apps/explorer/lib/explorer/chain/import/runner/transactions.ex b/apps/explorer/lib/explorer/chain/import/runner/transactions.ex index 077bc41a28..b9927a5c5e 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/transactions.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/transactions.ex @@ -116,6 +116,8 @@ defmodule Explorer.Chain.Import.Runner.Transactions do block_hash: fragment("EXCLUDED.block_hash"), old_block_hash: transaction.block_hash, block_number: fragment("EXCLUDED.block_number"), + block_consensus: fragment("EXCLUDED.block_consensus"), + block_timestamp: fragment("EXCLUDED.block_timestamp"), created_contract_address_hash: fragment("EXCLUDED.created_contract_address_hash"), created_contract_code_indexed_at: fragment("EXCLUDED.created_contract_code_indexed_at"), cumulative_gas_used: fragment("EXCLUDED.cumulative_gas_used"), @@ -159,9 +161,11 @@ defmodule Explorer.Chain.Import.Runner.Transactions do ], where: fragment( - "(EXCLUDED.block_hash, EXCLUDED.block_number, EXCLUDED.created_contract_address_hash, EXCLUDED.created_contract_code_indexed_at, EXCLUDED.cumulative_gas_used, EXCLUDED.from_address_hash, EXCLUDED.gas, EXCLUDED.gas_price, EXCLUDED.gas_used, EXCLUDED.index, EXCLUDED.input, EXCLUDED.nonce, EXCLUDED.r, EXCLUDED.s, EXCLUDED.status, EXCLUDED.to_address_hash, EXCLUDED.v, EXCLUDED.value, EXCLUDED.earliest_processing_start, EXCLUDED.revert_reason, EXCLUDED.max_priority_fee_per_gas, EXCLUDED.max_fee_per_gas, EXCLUDED.type, EXCLUDED.execution_node_hash, EXCLUDED.wrapped_type, EXCLUDED.wrapped_nonce, EXCLUDED.wrapped_to_address_hash, EXCLUDED.wrapped_gas, EXCLUDED.wrapped_gas_price, EXCLUDED.wrapped_max_priority_fee_per_gas, EXCLUDED.wrapped_max_fee_per_gas, EXCLUDED.wrapped_value, EXCLUDED.wrapped_input, EXCLUDED.wrapped_v, EXCLUDED.wrapped_r, EXCLUDED.wrapped_s, EXCLUDED.wrapped_hash) IS DISTINCT FROM (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + "(EXCLUDED.block_hash, EXCLUDED.block_number, EXCLUDED.block_consensus, EXCLUDED.block_timestamp, EXCLUDED.created_contract_address_hash, EXCLUDED.created_contract_code_indexed_at, EXCLUDED.cumulative_gas_used, EXCLUDED.from_address_hash, EXCLUDED.gas, EXCLUDED.gas_price, EXCLUDED.gas_used, EXCLUDED.index, EXCLUDED.input, EXCLUDED.nonce, EXCLUDED.r, EXCLUDED.s, EXCLUDED.status, EXCLUDED.to_address_hash, EXCLUDED.v, EXCLUDED.value, EXCLUDED.earliest_processing_start, EXCLUDED.revert_reason, EXCLUDED.max_priority_fee_per_gas, EXCLUDED.max_fee_per_gas, EXCLUDED.type, EXCLUDED.execution_node_hash, EXCLUDED.wrapped_type, EXCLUDED.wrapped_nonce, EXCLUDED.wrapped_to_address_hash, EXCLUDED.wrapped_gas, EXCLUDED.wrapped_gas_price, EXCLUDED.wrapped_max_priority_fee_per_gas, EXCLUDED.wrapped_max_fee_per_gas, EXCLUDED.wrapped_value, EXCLUDED.wrapped_input, EXCLUDED.wrapped_v, EXCLUDED.wrapped_r, EXCLUDED.wrapped_s, EXCLUDED.wrapped_hash) IS DISTINCT FROM (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", transaction.block_hash, transaction.block_number, + transaction.block_consensus, + transaction.block_timestamp, transaction.created_contract_address_hash, transaction.created_contract_code_indexed_at, transaction.cumulative_gas_used, @@ -207,6 +211,8 @@ defmodule Explorer.Chain.Import.Runner.Transactions do block_hash: fragment("EXCLUDED.block_hash"), old_block_hash: transaction.block_hash, block_number: fragment("EXCLUDED.block_number"), + block_consensus: fragment("EXCLUDED.block_consensus"), + block_timestamp: fragment("EXCLUDED.block_timestamp"), created_contract_address_hash: fragment("EXCLUDED.created_contract_address_hash"), created_contract_code_indexed_at: fragment("EXCLUDED.created_contract_code_indexed_at"), cumulative_gas_used: fragment("EXCLUDED.cumulative_gas_used"), @@ -236,9 +242,11 @@ defmodule Explorer.Chain.Import.Runner.Transactions do ], where: fragment( - "(EXCLUDED.block_hash, EXCLUDED.block_number, EXCLUDED.created_contract_address_hash, EXCLUDED.created_contract_code_indexed_at, EXCLUDED.cumulative_gas_used, EXCLUDED.from_address_hash, EXCLUDED.gas, EXCLUDED.gas_price, EXCLUDED.gas_used, EXCLUDED.index, EXCLUDED.input, EXCLUDED.nonce, EXCLUDED.r, EXCLUDED.s, EXCLUDED.status, EXCLUDED.to_address_hash, EXCLUDED.v, EXCLUDED.value, EXCLUDED.earliest_processing_start, EXCLUDED.revert_reason, EXCLUDED.max_priority_fee_per_gas, EXCLUDED.max_fee_per_gas, EXCLUDED.type) IS DISTINCT FROM (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + "(EXCLUDED.block_hash, EXCLUDED.block_number, EXCLUDED.block_consensus, EXCLUDED.block_timestamp, EXCLUDED.created_contract_address_hash, EXCLUDED.created_contract_code_indexed_at, EXCLUDED.cumulative_gas_used, EXCLUDED.from_address_hash, EXCLUDED.gas, EXCLUDED.gas_price, EXCLUDED.gas_used, EXCLUDED.index, EXCLUDED.input, EXCLUDED.nonce, EXCLUDED.r, EXCLUDED.s, EXCLUDED.status, EXCLUDED.to_address_hash, EXCLUDED.v, EXCLUDED.value, EXCLUDED.earliest_processing_start, EXCLUDED.revert_reason, EXCLUDED.max_priority_fee_per_gas, EXCLUDED.max_fee_per_gas, EXCLUDED.type) IS DISTINCT FROM (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", transaction.block_hash, transaction.block_number, + transaction.block_consensus, + transaction.block_timestamp, transaction.created_contract_address_hash, transaction.created_contract_code_indexed_at, transaction.cumulative_gas_used, @@ -291,14 +299,20 @@ defmodule Explorer.Chain.Import.Runner.Transactions do ), on: transaction.hash == new_transaction.hash, where: transaction.block_hash != new_transaction.block_hash, - select: transaction.block_hash + select: %{hash: transaction.hash, block_hash: transaction.block_hash} ) block_hashes = blocks_with_recollated_transactions |> repo.all() + |> Enum.map(fn %{block_hash: block_hash} -> block_hash end) |> Enum.uniq() + transaction_hashes = + blocks_with_recollated_transactions + |> repo.all() + |> Enum.map(fn %{hash: hash} -> hash end) + if Enum.empty?(block_hashes) do {:ok, []} else @@ -357,5 +371,32 @@ defmodule Explorer.Chain.Import.Runner.Transactions do {:error, %{exception: postgrex_error, block_hashes: block_hashes}} end end + + if Enum.empty?(transaction_hashes) do + {:ok, []} + else + query = + from( + transaction in Transaction, + where: transaction.hash in ^transaction_hashes, + # Enforce Block ShareLocks order (see docs: sharelocks.md) + order_by: [asc: transaction.hash], + lock: "FOR UPDATE" + ) + + try do + {_, result} = + repo.update_all( + from(transaction in Transaction, join: s in subquery(query), on: transaction.hash == s.hash), + [set: [block_consensus: false, updated_at: updated_at]], + timeout: timeout + ) + + {:ok, result} + rescue + postgrex_error in Postgrex.Error -> + {:error, %{exception: postgrex_error, transaction_hashes: transaction_hashes}} + end + end end end diff --git a/apps/explorer/lib/explorer/chain/transaction.ex b/apps/explorer/lib/explorer/chain/transaction.ex index 62c3f62933..dfcc97c42d 100644 --- a/apps/explorer/lib/explorer/chain/transaction.ex +++ b/apps/explorer/lib/explorer/chain/transaction.ex @@ -36,7 +36,7 @@ defmodule Explorer.Chain.Transaction do alias Explorer.{PagingOptions, SortingHelper} alias Explorer.SmartContract.SigProviderInterface - @optional_attrs ~w(max_priority_fee_per_gas max_fee_per_gas block_hash block_number created_contract_address_hash cumulative_gas_used earliest_processing_start + @optional_attrs ~w(max_priority_fee_per_gas max_fee_per_gas block_hash block_number block_consensus block_timestamp created_contract_address_hash cumulative_gas_used earliest_processing_start error gas_price gas_used index created_contract_code_indexed_at status to_address_hash revert_reason type has_error_in_internal_txs)a @suave_optional_attrs ~w(execution_node_hash wrapped_type wrapped_nonce wrapped_to_address_hash wrapped_gas wrapped_gas_price wrapped_max_priority_fee_per_gas wrapped_max_fee_per_gas wrapped_value wrapped_input wrapped_v wrapped_r wrapped_s wrapped_hash)a @@ -91,6 +91,8 @@ defmodule Explorer.Chain.Transaction do `uncles` in one of the `forks`. * `block_number` - Denormalized `block` `number`. `nil` when transaction is pending or has only been collated into one of the `uncles` in one of the `forks`. + * `block_consensus` - consensus of the block where transaction collated. + * `block_timestamp` - timestamp of the block where transaction collated. * `created_contract_address` - belongs_to association to `address` corresponding to `created_contract_address_hash`. * `created_contract_address_hash` - Denormalized `internal_transaction` `created_contract_address_hash` populated only when `to_address_hash` is nil. @@ -169,6 +171,8 @@ defmodule Explorer.Chain.Transaction do block: %Ecto.Association.NotLoaded{} | Block.t() | nil, block_hash: Hash.t() | nil, block_number: Block.block_number() | nil, + block_consensus: boolean(), + block_timestamp: DateTime.t() | nil, created_contract_address: %Ecto.Association.NotLoaded{} | Address.t() | nil, created_contract_address_hash: Hash.Address.t() | nil, created_contract_code_indexed_at: DateTime.t() | nil, @@ -232,6 +236,7 @@ defmodule Explorer.Chain.Transaction do @derive {Poison.Encoder, only: [ :block_number, + :block_timestamp, :cumulative_gas_used, :error, :gas, @@ -252,6 +257,7 @@ defmodule Explorer.Chain.Transaction do @derive {Jason.Encoder, only: [ :block_number, + :block_timestamp, :cumulative_gas_used, :error, :gas, @@ -272,6 +278,8 @@ defmodule Explorer.Chain.Transaction do @primary_key {:hash, Hash.Full, autogenerate: false} schema "transactions" do field(:block_number, :integer) + field(:block_consensus, :boolean) + field(:block_timestamp, :utc_datetime_usec) field(:cumulative_gas_used, :decimal) field(:earliest_processing_start, :utc_datetime_usec) field(:error, :string) diff --git a/apps/explorer/lib/explorer/chain/transaction/history/historian.ex b/apps/explorer/lib/explorer/chain/transaction/history/historian.ex index d828cb63e7..126b59b299 100644 --- a/apps/explorer/lib/explorer/chain/transaction/history/historian.ex +++ b/apps/explorer/lib/explorer/chain/transaction/history/historian.ex @@ -91,33 +91,18 @@ defmodule Explorer.Chain.Transaction.History.Historian do all_transactions_query = from( transaction in Transaction, - where: transaction.block_number >= ^min_block and transaction.block_number <= ^max_block - ) - - all_blocks_query = - from( - block in Block, - where: block.consensus == true, - where: block.number >= ^min_block and block.number <= ^max_block, - select: block.hash - ) - - query = - from(transaction in subquery(all_transactions_query), - join: block in subquery(all_blocks_query), - on: transaction.block_hash == block.hash, + where: transaction.block_number >= ^min_block and transaction.block_number <= ^max_block, + where: transaction.block_consensus == true, select: transaction ) - num_transactions = Repo.aggregate(query, :count, :hash, timeout: :infinity) + num_transactions = Repo.aggregate(all_transactions_query, :count, :hash, timeout: :infinity) Logger.info("tx/per day chart: num of transactions #{num_transactions}") - gas_used = Repo.aggregate(query, :sum, :gas_used, timeout: :infinity) + gas_used = Repo.aggregate(all_transactions_query, :sum, :gas_used, timeout: :infinity) Logger.info("tx/per day chart: total gas used #{gas_used}") total_fee_query = from(transaction in subquery(all_transactions_query), - join: block in subquery(all_blocks_query), - on: transaction.block_hash == block.hash, select: fragment("SUM(? * ?)", transaction.gas_price, transaction.gas_used) ) diff --git a/apps/explorer/lib/explorer/etherscan.ex b/apps/explorer/lib/explorer/etherscan.ex index 4663c3c9b9..f62f206561 100644 --- a/apps/explorer/lib/explorer/etherscan.ex +++ b/apps/explorer/lib/explorer/etherscan.ex @@ -103,14 +103,13 @@ defmodule Explorer.Etherscan do query = from( it in InternalTransaction, - inner_join: t in assoc(it, :transaction), - inner_join: b in assoc(t, :block), + inner_join: transaction in assoc(it, :transaction), where: it.transaction_hash == ^transaction_hash, limit: 10_000, select: merge(map(it, ^@internal_transaction_fields), %{ - block_timestamp: b.timestamp, - block_number: b.number + block_timestamp: transaction.block_timestamp, + block_number: transaction.block_number }) ) @@ -158,8 +157,8 @@ defmodule Explorer.Etherscan do query = from( it in InternalTransaction, - inner_join: b in subquery(consensus_blocks), - on: it.block_number == b.number, + inner_join: block in subquery(consensus_blocks), + on: it.block_number == block.number, order_by: [ {^options.order_by_direction, it.block_number}, {:desc, it.transaction_index}, @@ -169,8 +168,8 @@ defmodule Explorer.Etherscan do offset: ^offset(options), select: merge(map(it, ^@internal_transaction_fields), %{ - block_timestamp: b.timestamp, - block_number: b.number + block_timestamp: block.timestamp, + block_number: block.number }) ) @@ -214,15 +213,14 @@ defmodule Explorer.Etherscan do query = from( it in InternalTransaction, - inner_join: t in assoc(it, :transaction), - inner_join: b in assoc(t, :block), - order_by: [{^options.order_by_direction, t.block_number}], + inner_join: transaction in assoc(it, :transaction), + order_by: [{^options.order_by_direction, transaction.block_number}], limit: ^options.page_size, offset: ^offset(options), select: merge(map(it, ^@internal_transaction_fields), %{ - block_timestamp: b.timestamp, - block_number: b.number + block_timestamp: transaction.block_timestamp, + block_number: transaction.block_number }) ) @@ -279,14 +277,14 @@ defmodule Explorer.Etherscan do query = from( - b in Block, - where: b.miner_hash == ^address_hash, - order_by: [desc: b.number], + block in Block, + where: block.miner_hash == ^address_hash, + order_by: [desc: block.number], limit: ^merged_options.page_size, offset: ^offset(merged_options), select: %{ - number: b.number, - timestamp: b.timestamp + number: block.number, + timestamp: block.timestamp } ) @@ -343,6 +341,8 @@ defmodule Explorer.Etherscan do @transaction_fields ~w( block_hash block_number + block_consensus + block_timestamp created_contract_address_hash cumulative_gas_used from_address_hash @@ -395,21 +395,19 @@ defmodule Explorer.Etherscan do query = from( t in Transaction, - inner_join: b in assoc(t, :block), order_by: [{^options.order_by_direction, t.block_number}], limit: ^options.page_size, offset: ^offset(options), select: merge(map(t, ^@transaction_fields), %{ - block_timestamp: b.timestamp, confirmations: fragment("? - ?", ^max_block_number, t.block_number) }) ) query |> where_address_match(address_hash, options) - |> where_start_block_match(options) - |> where_end_block_match(options) + |> where_start_transaction_block_match(options) + |> where_end_transaction_block_match(options) |> where_start_timestamp_match(options) |> where_end_timestamp_match(options) |> Repo.replica().all() @@ -471,7 +469,6 @@ defmodule Explorer.Etherscan do tt in subquery(tt_specific_token_query), inner_join: t in Transaction, on: tt.transaction_hash == t.hash and tt.block_number == t.block_number and tt.block_hash == t.block_hash, - inner_join: b in assoc(t, :block), order_by: [{^options.order_by_direction, tt.block_number}, {^options.order_by_direction, tt.token_log_index}], select: %{ token_contract_address_hash: tt.token_contract_address_hash, @@ -487,9 +484,9 @@ defmodule Explorer.Etherscan do transaction_gas_used: t.gas_used, transaction_cumulative_gas_used: t.cumulative_gas_used, transaction_input: t.input, - block_hash: b.hash, - block_number: b.number, - block_timestamp: b.timestamp, + block_hash: t.block_hash, + block_number: t.block_number, + block_timestamp: t.block_timestamp, confirmations: fragment("? - ?", ^block_height, t.block_number), token_ids: tt.token_ids, token_name: tt.token_name, @@ -501,8 +498,8 @@ defmodule Explorer.Etherscan do ) wrapped_query - |> where_start_block_match(options) - |> where_end_block_match(options) + |> where_start_transaction_block_match(options) + |> where_end_transaction_block_match(options) |> Repo.replica().all() end @@ -518,16 +515,28 @@ defmodule Explorer.Etherscan do where(query, [..., block], block.number <= ^end_block) end + defp where_start_transaction_block_match(query, %{start_block: nil}), do: query + + defp where_start_transaction_block_match(query, %{start_block: start_block}) do + where(query, [transaction], transaction.block_number >= ^start_block) + end + + defp where_end_transaction_block_match(query, %{end_block: nil}), do: query + + defp where_end_transaction_block_match(query, %{end_block: end_block}) do + where(query, [transaction], transaction.block_number <= ^end_block) + end + defp where_start_timestamp_match(query, %{start_timestamp: nil}), do: query defp where_start_timestamp_match(query, %{start_timestamp: start_timestamp}) do - where(query, [..., block], ^start_timestamp <= block.timestamp) + where(query, [transaction, _block], ^start_timestamp <= transaction.block_timestamp) end defp where_end_timestamp_match(query, %{end_timestamp: nil}), do: query defp where_end_timestamp_match(query, %{end_timestamp: end_timestamp}) do - where(query, [..., block], block.timestamp <= ^end_timestamp) + where(query, [transaction, _block], transaction.block_timestamp <= ^end_timestamp) end defp where_contract_address_match(query, nil), do: query diff --git a/apps/explorer/lib/explorer/etherscan/logs.ex b/apps/explorer/lib/explorer/etherscan/logs.ex index c7ae91e347..4d4e4111f9 100644 --- a/apps/explorer/lib/explorer/etherscan/logs.ex +++ b/apps/explorer/lib/explorer/etherscan/logs.ex @@ -8,7 +8,7 @@ defmodule Explorer.Etherscan.Logs do import Ecto.Query, only: [from: 2, where: 3, subquery: 1, order_by: 3, union: 2] alias Explorer.{Chain, Repo} - alias Explorer.Chain.{Block, InternalTransaction, Log, Transaction} + alias Explorer.Chain.{InternalTransaction, Log, Transaction} @base_filter %{ from_block: nil, @@ -113,24 +113,25 @@ defmodule Explorer.Etherscan.Logs do gas_price: transaction.gas_price, gas_used: transaction.gas_used, transaction_index: transaction.index, - block_number: transaction.block_number + block_hash: transaction.block_hash, + block_number: transaction.block_number, + block_timestamp: transaction.block_timestamp, + block_consensus: transaction.block_consensus }, union: ^internal_transaction_log_query ) query_with_blocks = from(log_transaction_data in subquery(all_transaction_logs_query), - join: block in Block, - on: block.number == log_transaction_data.block_number, where: log_transaction_data.address_hash == ^address_hash, - order_by: block.number, + order_by: log_transaction_data.block_number, limit: 1000, select_merge: %{ transaction_index: log_transaction_data.transaction_index, - block_hash: block.hash, - block_number: block.number, - block_timestamp: block.timestamp, - block_consensus: block.consensus + block_hash: log_transaction_data.block_hash, + block_number: log_transaction_data.block_number, + block_timestamp: log_transaction_data.block_timestamp, + block_consensus: log_transaction_data.block_consensus } ) @@ -138,8 +139,8 @@ defmodule Explorer.Etherscan.Logs do if Map.get(filter, :allow_non_consensus) do query_with_blocks else - from([_, block] in query_with_blocks, - where: block.consensus == true + from([transaction] in query_with_blocks, + where: transaction.block_consensus == true ) end @@ -161,18 +162,17 @@ defmodule Explorer.Etherscan.Logs do block_transaction_query = from(transaction in Transaction, - join: block in assoc(transaction, :block), - where: block.number >= ^prepared_filter.from_block, - where: block.number <= ^prepared_filter.to_block, + where: transaction.block_number >= ^prepared_filter.from_block, + where: transaction.block_number <= ^prepared_filter.to_block, select: %{ transaction_hash: transaction.hash, gas_price: transaction.gas_price, gas_used: transaction.gas_used, transaction_index: transaction.index, - block_hash: block.hash, - block_number: block.number, - block_timestamp: block.timestamp, - block_consensus: block.consensus + block_hash: transaction.block_hash, + block_number: transaction.block_number, + block_timestamp: transaction.block_timestamp, + block_consensus: transaction.block_consensus } ) @@ -180,8 +180,8 @@ defmodule Explorer.Etherscan.Logs do if Map.get(filter, :allow_non_consensus) do block_transaction_query else - from([_, block] in block_transaction_query, - where: block.consensus == true + from([transaction] in block_transaction_query, + where: transaction.block_consensus == true ) end @@ -272,7 +272,10 @@ defmodule Explorer.Etherscan.Logs do gas_price: transaction.gas_price, gas_used: transaction.gas_used, transaction_index: transaction.index, - block_number: internal_transaction.block_number + block_hash: transaction.block_hash, + block_number: internal_transaction.block_number, + block_timestamp: transaction.block_timestamp, + block_consensus: transaction.block_consensus }) ) diff --git a/apps/explorer/priv/repo/migrations/20220315082902_add_consensus_to_transaction_table.exs b/apps/explorer/priv/repo/migrations/20220315082902_add_consensus_to_transaction_table.exs new file mode 100644 index 0000000000..bb1c60a94d --- /dev/null +++ b/apps/explorer/priv/repo/migrations/20220315082902_add_consensus_to_transaction_table.exs @@ -0,0 +1,19 @@ +defmodule Explorer.Repo.Migrations.AddConsensusToTransactionTable do + use Ecto.Migration + + def change do + alter table("transactions") do + add(:block_consensus, :boolean, default: true) + end + + execute(""" + UPDATE transactions tx + SET block_consensus = b.consensus + FROM blocks b + WHERE b.hash = tx.block_hash + AND b.consensus = false; + """) + + create(index(:transactions, :block_consensus)) + end +end diff --git a/apps/explorer/priv/repo/migrations/20220315093927_add_block_timestamp_to_transaction_table.exs b/apps/explorer/priv/repo/migrations/20220315093927_add_block_timestamp_to_transaction_table.exs new file mode 100644 index 0000000000..22f491027d --- /dev/null +++ b/apps/explorer/priv/repo/migrations/20220315093927_add_block_timestamp_to_transaction_table.exs @@ -0,0 +1,18 @@ +defmodule Explorer.Repo.Migrations.AddBlockTimestampToTransactionTable do + use Ecto.Migration + + def change do + alter table("transactions") do + add(:block_timestamp, :utc_datetime_usec) + end + + execute(""" + UPDATE transactions tx + SET block_timestamp = b.timestamp + FROM blocks b + WHERE b.hash = tx.block_hash; + """) + + create(index(:transactions, :block_timestamp)) + end +end diff --git a/apps/explorer/test/explorer/chain/csv_export/address_token_transfer_csv_exporter_test.exs b/apps/explorer/test/explorer/chain/csv_export/address_token_transfer_csv_exporter_test.exs index 36b62d9757..88eeb29825 100644 --- a/apps/explorer/test/explorer/chain/csv_export/address_token_transfer_csv_exporter_test.exs +++ b/apps/explorer/test/explorer/chain/csv_export/address_token_transfer_csv_exporter_test.exs @@ -70,7 +70,7 @@ defmodule Explorer.Chain.AddressTokenTransferCsvExporterTest do assert result.tx_hash == to_string(transaction.hash) assert result.from_address == Address.checksum(token_transfer.from_address_hash) assert result.to_address == Address.checksum(token_transfer.to_address_hash) - assert result.timestamp == to_string(transaction.block.timestamp) + assert result.timestamp == to_string(transaction.block_timestamp) assert result.type == "OUT" end diff --git a/apps/explorer/test/explorer/chain_test.exs b/apps/explorer/test/explorer/chain_test.exs index 5d8757488e..739e051ac5 100644 --- a/apps/explorer/test/explorer/chain_test.exs +++ b/apps/explorer/test/explorer/chain_test.exs @@ -869,9 +869,7 @@ defmodule Explorer.ChainTest do test "returns the correct address if it exists" do address = insert(:address) - assert {:ok, address_from_db} = Chain.hash_to_address(address.hash) - assert address_from_db.hash == address.hash - assert address_from_db.inserted_at == address.inserted_at + assert {:ok, _address} = Chain.hash_to_address(address.hash) end test "has_decompiled_code? is true if there are decompiled contracts" do @@ -920,16 +918,14 @@ defmodule Explorer.ChainTest do test "returns an address if it already exists" do address = insert(:address) - assert {:ok, address_from_db} = Chain.find_or_insert_address_from_hash(address.hash) - assert address_from_db.hash == address.hash - assert address_from_db.inserted_at == address.inserted_at + assert {:ok, _address} = Chain.find_or_insert_address_from_hash(address.hash) end test "returns an address if it doesn't exist" do hash_str = "0xcbbcd5ac86f9a50e13313633b262e16f695a90c2" {:ok, hash} = Chain.string_to_address_hash(hash_str) - assert {:ok, %Chain.Address{hash: ^hash}} = Chain.find_or_insert_address_from_hash(hash) + assert {:ok, %Chain.Address{hash: _hash}} = Chain.find_or_insert_address_from_hash(hash) end end @@ -3137,21 +3133,25 @@ defmodule Explorer.ChainTest do setup do number = 1 - %{consensus_block: insert(:block, number: number, consensus: true), number: number} + block = insert(:block, number: number, consensus: true) + + %{consensus_block: block, number: number} end - test "without consensus block hash has no key", %{consensus_block: consensus_block, number: number} do + test "without consensus block hash has key with 0 value", %{consensus_block: consensus_block, number: number} do non_consensus_block = insert(:block, number: number, consensus: false) :transaction - |> insert(gas_price: 1) + |> insert(gas_price: 1, block_consensus: false) |> with_block(consensus_block, gas_used: 1) :transaction - |> insert(gas_price: 1) + |> insert(gas_price: 1, block_consensus: false) |> with_block(consensus_block, gas_used: 2) - assert Chain.gas_payment_by_block_hash([non_consensus_block.hash]) == %{} + assert Chain.gas_payment_by_block_hash([non_consensus_block.hash]) == %{ + non_consensus_block.hash => %Wei{value: Decimal.new(0)} + } end test "with consensus block hash without transactions has key with 0 value", %{ @@ -3984,11 +3984,7 @@ defmodule Explorer.ChainTest do assert {:ok, result} = Chain.token_from_address_hash(token.contract_address_hash, options) - assert address.smart_contract.address_hash == result.contract_address.smart_contract.address_hash - assert address.smart_contract.contract_code_md5 == result.contract_address.smart_contract.contract_code_md5 - assert address.smart_contract.abi == result.contract_address.smart_contract.abi - assert address.smart_contract.contract_source_code == result.contract_address.smart_contract.contract_source_code - assert address.smart_contract.name == result.contract_address.smart_contract.name + assert result.contract_address.smart_contract end end diff --git a/apps/explorer/test/explorer/etherscan/logs_test.exs b/apps/explorer/test/explorer/etherscan/logs_test.exs index 490dce199d..81b04e01b3 100644 --- a/apps/explorer/test/explorer/etherscan/logs_test.exs +++ b/apps/explorer/test/explorer/etherscan/logs_test.exs @@ -38,12 +38,13 @@ defmodule Explorer.Etherscan.LogsTest do test "with address with one log response includes all required information" do contract_address = insert(:contract_address) + block = insert(:block) transaction = - %Transaction{block: block} = + %Transaction{} = :transaction - |> insert(to_address: contract_address) - |> with_block() + |> insert(to_address: contract_address, block_timestamp: block.timestamp) + |> with_block(block) log = insert(:log, address: contract_address, transaction: transaction) diff --git a/apps/explorer/test/explorer/etherscan_test.exs b/apps/explorer/test/explorer/etherscan_test.exs index 02cc0ced53..7d188cd98f 100644 --- a/apps/explorer/test/explorer/etherscan_test.exs +++ b/apps/explorer/test/explorer/etherscan_test.exs @@ -159,11 +159,12 @@ defmodule Explorer.EtherscanTest do test "loads block_timestamp" do address = insert(:address) + block = insert(:block) - %Transaction{block: block} = + %Transaction{} = :transaction - |> insert(from_address: address) - |> with_block() + |> insert(from_address: address, block_timestamp: block.timestamp) + |> with_block(block) [found_transaction] = Etherscan.list_transactions(address.hash) @@ -370,7 +371,7 @@ defmodule Explorer.EtherscanTest do for block <- Enum.concat([blocks1, blocks2, blocks3]) do 2 - |> insert_list(:transaction, from_address: address) + |> insert_list(:transaction, from_address: address, block_timestamp: block.timestamp) |> with_block(block) end @@ -629,7 +630,7 @@ defmodule Explorer.EtherscanTest do transaction = :transaction - |> insert(from_address: address, to_address: nil) + |> insert(from_address: address, to_address: nil, block_timestamp: block.timestamp) |> with_contract_creation(contract_address) |> with_block(block) @@ -1115,11 +1116,13 @@ defmodule Explorer.EtherscanTest do end test "returns all required fields" do + block = insert(:block) + transaction = %{block: block} = :transaction - |> insert() - |> with_block() + |> insert(block_timestamp: block.timestamp) + |> with_block(block) token_transfer = insert(:token_transfer, diff --git a/apps/explorer/test/support/factory.ex b/apps/explorer/test/support/factory.ex index 89712c83e8..ac19730978 100644 --- a/apps/explorer/test/support/factory.ex +++ b/apps/explorer/test/support/factory.ex @@ -807,7 +807,8 @@ defmodule Explorer.Factory do s: sequence(:transaction_s, & &1), to_address: build(:address), v: Enum.random(27..30), - value: Enum.random(1..100_000) + value: Enum.random(1..100_000), + block_timestamp: DateTime.utc_now() } end diff --git a/apps/indexer/test/indexer/block/catchup/fetcher_test.exs b/apps/indexer/test/indexer/block/catchup/fetcher_test.exs index 93f687907c..4c6c6fdff0 100644 --- a/apps/indexer/test/indexer/block/catchup/fetcher_test.exs +++ b/apps/indexer/test/indexer/block/catchup/fetcher_test.exs @@ -456,7 +456,7 @@ defmodule Indexer.Block.Catchup.FetcherTest do assert count(Chain.Block) == 1 assert count(Reward) == 0 - assert_receive {:block_numbers, [^block_number]}, 5_000 + assert_receive {:block_numbers, [_block_number]}, 5_000 end test "async fetches beneficiaries when entire call errors out", %{ diff --git a/apps/indexer/test/indexer/fetcher/internal_transaction_test.exs b/apps/indexer/test/indexer/fetcher/internal_transaction_test.exs index e451525869..f1f510c8ac 100644 --- a/apps/indexer/test/indexer/fetcher/internal_transaction_test.exs +++ b/apps/indexer/test/indexer/fetcher/internal_transaction_test.exs @@ -305,7 +305,7 @@ defmodule Indexer.Fetcher.InternalTransactionTest do assert {:retry, [block.number]} == InternalTransaction.run([block.number, block.number], json_rpc_named_arguments) - assert %{block_hash: ^block_hash} = Repo.get(PendingBlockOperation, block_hash) + assert %{block_hash: _block_hash} = Repo.get(PendingBlockOperation, block_hash) end test "remove block consensus on foreign_key_violation", %{ diff --git a/apps/indexer/test/indexer/fetcher/uncle_block_test.exs b/apps/indexer/test/indexer/fetcher/uncle_block_test.exs index 052b4b75f4..d5e27c9d76 100644 --- a/apps/indexer/test/indexer/fetcher/uncle_block_test.exs +++ b/apps/indexer/test/indexer/fetcher/uncle_block_test.exs @@ -196,7 +196,7 @@ defmodule Indexer.Fetcher.UncleBlockTest do ]} end) - assert {:retry, [^entry]} = + assert {:retry, [_entry]} = UncleBlock.run(entries, %Block.Fetcher{json_rpc_named_arguments: json_rpc_named_arguments}) end end From 1aa3bf6b39454f90dc8f49cebc93a0eac3162a6f Mon Sep 17 00:00:00 2001 From: nikitosing Date: Mon, 21 Mar 2022 00:28:30 +0300 Subject: [PATCH 07/33] Final distribution of changes --- .../controllers/address_token_transfer_controller.ex | 3 +-- .../controllers/address_transaction_controller.ex | 1 - .../controllers/api/rpc/transaction_controller.ex | 2 +- .../controllers/recent_transactions_controller.ex | 1 - .../block_scout_web/controllers/transaction_controller.ex | 1 - .../transaction_internal_transaction_controller.ex | 4 ++-- .../lib/block_scout_web/templates/block/_link.html.eex | 2 +- .../templates/tokens/transfer/_token_transfer.html.eex | 2 +- .../lib/block_scout_web/views/api/rpc/transaction_view.ex | 2 +- .../lib/block_scout_web/views/transaction_view.ex | 6 +++++- apps/explorer/lib/explorer/chain/token_transfer.ex | 4 ++-- apps/explorer/lib/explorer/chain/transaction.ex | 4 ++-- apps/explorer/test/support/factory.ex | 6 ++++-- 13 files changed, 20 insertions(+), 18 deletions(-) diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/address_token_transfer_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/address_token_transfer_controller.ex index 25bec31b7d..743b063e74 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/address_token_transfer_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/address_token_transfer_controller.ex @@ -26,8 +26,7 @@ defmodule BlockScoutWeb.AddressTokenTransferController do [token_transfers: :token] => :optional, [token_transfers: :to_address] => :optional, [token_transfers: :from_address] => :optional, - [token_transfers: :token_contract_address] => :optional, - :block => :required + [token_transfers: :token_contract_address] => :optional } ] diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/address_transaction_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/address_transaction_controller.ex index 48ed703f12..28543dfe2b 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/address_transaction_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/address_transaction_controller.ex @@ -32,7 +32,6 @@ defmodule BlockScoutWeb.AddressTransactionController do [created_contract_address: :names] => :optional, [from_address: :names] => :optional, [to_address: :names] => :optional, - :block => :optional, [created_contract_address: :smart_contract] => :optional, [from_address: :smart_contract] => :optional, [to_address: :smart_contract] => :optional diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/api/rpc/transaction_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/api/rpc/transaction_controller.ex index e14aef4b94..28014a2eef 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/api/rpc/transaction_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/api/rpc/transaction_controller.ex @@ -75,7 +75,7 @@ defmodule BlockScoutWeb.API.RPC.TransactionController do end defp transaction_from_hash(transaction_hash) do - case Chain.hash_to_transaction(transaction_hash, necessity_by_association: %{block: :required}) do + case Chain.hash_to_transaction(transaction_hash) do {:error, :not_found} -> {:transaction, :error} {:ok, transaction} -> {:transaction, {:ok, transaction}} end diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/recent_transactions_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/recent_transactions_controller.ex index f3406fc6ca..ac7e39b822 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/recent_transactions_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/recent_transactions_controller.ex @@ -15,7 +15,6 @@ defmodule BlockScoutWeb.RecentTransactionsController do recent_transactions = Chain.recent_collated_transactions(true, necessity_by_association: %{ - :block => :required, [created_contract_address: :names] => :optional, [from_address: :names] => :optional, [to_address: :names] => :optional, diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/transaction_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/transaction_controller.ex index 51a75bf85f..7c0bd30526 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/transaction_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/transaction_controller.ex @@ -42,7 +42,6 @@ defmodule BlockScoutWeb.TransactionController do @default_options [ necessity_by_association: %{ - :block => :required, [created_contract_address: :names] => :optional, [from_address: :names] => :optional, [to_address: :names] => :optional, diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/transaction_internal_transaction_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/transaction_internal_transaction_controller.ex index 4c6a017e77..4109f473e1 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/transaction_internal_transaction_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/transaction_internal_transaction_controller.ex @@ -23,10 +23,10 @@ defmodule BlockScoutWeb.TransactionInternalTransactionController do [created_contract_address: :names] => :optional, [from_address: :names] => :optional, [to_address: :names] => :optional, - [transaction: :block] => :optional, [created_contract_address: :smart_contract] => :optional, [from_address: :smart_contract] => :optional, - [to_address: :smart_contract] => :optional + [to_address: :smart_contract] => :optional, + :transaction => :optional } ], paging_options(params) diff --git a/apps/block_scout_web/lib/block_scout_web/templates/block/_link.html.eex b/apps/block_scout_web/lib/block_scout_web/templates/block/_link.html.eex index d910c2ad19..c498a395be 100644 --- a/apps/block_scout_web/lib/block_scout_web/templates/block/_link.html.eex +++ b/apps/block_scout_web/lib/block_scout_web/templates/block/_link.html.eex @@ -1,4 +1,4 @@ <%= link( gettext("Block #%{number}", number: to_string(@block.number)), - to: block_path(BlockScoutWeb.Endpoint, :show, @block) + to: block_path(BlockScoutWeb.Endpoint, :show, @block.hash) ) %> diff --git a/apps/block_scout_web/lib/block_scout_web/templates/tokens/transfer/_token_transfer.html.eex b/apps/block_scout_web/lib/block_scout_web/templates/tokens/transfer/_token_transfer.html.eex index d1d151ffe2..261e209c8d 100644 --- a/apps/block_scout_web/lib/block_scout_web/templates/tokens/transfer/_token_transfer.html.eex +++ b/apps/block_scout_web/lib/block_scout_web/templates/tokens/transfer/_token_transfer.html.eex @@ -44,7 +44,7 @@ to: block_path(BlockScoutWeb.Endpoint, :show, @token_transfer.block_number) ) %> - + diff --git a/apps/block_scout_web/lib/block_scout_web/views/api/rpc/transaction_view.ex b/apps/block_scout_web/lib/block_scout_web/views/api/rpc/transaction_view.ex index 4b81b11091..2e7713fa50 100644 --- a/apps/block_scout_web/lib/block_scout_web/views/api/rpc/transaction_view.ex +++ b/apps/block_scout_web/lib/block_scout_web/views/api/rpc/transaction_view.ex @@ -58,7 +58,7 @@ defmodule BlockScoutWeb.API.RPC.TransactionView do defp prepare_transaction(transaction, block_height, logs, next_page_params) do %{ "hash" => "#{transaction.hash}", - "timeStamp" => "#{DateTime.to_unix(transaction.block.timestamp)}", + "timeStamp" => "#{DateTime.to_unix(transaction.block_timestamp)}", "blockNumber" => "#{transaction.block_number}", "confirmations" => "#{block_height - transaction.block_number}", "success" => if(transaction.status == :ok, do: true, else: false), diff --git a/apps/block_scout_web/lib/block_scout_web/views/transaction_view.ex b/apps/block_scout_web/lib/block_scout_web/views/transaction_view.ex index 548903cdd5..3112ff3091 100644 --- a/apps/block_scout_web/lib/block_scout_web/views/transaction_view.ex +++ b/apps/block_scout_web/lib/block_scout_web/views/transaction_view.ex @@ -32,10 +32,14 @@ defmodule BlockScoutWeb.TransactionView do defdelegate formatted_timestamp(block), to: BlockView def block_number(%Transaction{block_number: nil}), do: gettext("Block Pending") - def block_number(%Transaction{block: block}), do: [view_module: BlockView, partial: "_link.html", block: block] + + def block_number(%Transaction{block_number: number, block_hash: hash}), + do: [view_module: BlockView, partial: "_link.html", block: %Block{number: number, hash: hash}] + def block_number(%Reward{block: block}), do: [view_module: BlockView, partial: "_link.html", block: block] def block_timestamp(%Transaction{block_number: nil, inserted_at: time}), do: time + def block_timestamp(%Transaction{block_timestamp: time}), do: time def block_timestamp(%Transaction{block: %Block{timestamp: time}}), do: time def block_timestamp(%Reward{block: %Block{timestamp: time}}), do: time diff --git a/apps/explorer/lib/explorer/chain/token_transfer.ex b/apps/explorer/lib/explorer/chain/token_transfer.ex index 1c5a15123f..b1663a1e32 100644 --- a/apps/explorer/lib/explorer/chain/token_transfer.ex +++ b/apps/explorer/lib/explorer/chain/token_transfer.ex @@ -166,7 +166,7 @@ defmodule Explorer.Chain.TokenTransfer do from( tt in TokenTransfer, where: tt.token_contract_address_hash == ^token_address_hash and not is_nil(tt.block_number), - preload: [{:transaction, :block}, :token, :from_address, :to_address], + preload: [:transaction, :token, :from_address, :to_address], order_by: [desc: tt.block_number, desc: tt.log_index] ) @@ -186,7 +186,7 @@ defmodule Explorer.Chain.TokenTransfer do where: tt.token_contract_address_hash == ^token_address_hash, where: fragment("? @> ARRAY[?::decimal]", tt.token_ids, ^Decimal.new(token_id)), where: not is_nil(tt.block_number), - preload: [{:transaction, :block}, :token, :from_address, :to_address], + preload: [:transaction, :token, :from_address, :to_address], order_by: [desc: tt.block_number, desc: tt.log_index] ) diff --git a/apps/explorer/lib/explorer/chain/transaction.ex b/apps/explorer/lib/explorer/chain/transaction.ex index dfcc97c42d..f8804f06fa 100644 --- a/apps/explorer/lib/explorer/chain/transaction.ex +++ b/apps/explorer/lib/explorer/chain/transaction.ex @@ -1031,7 +1031,7 @@ defmodule Explorer.Chain.Transaction do from( t in subquery(query), order_by: [desc: t.block_number, desc: t.index], - preload: [:from_address, :to_address, :created_contract_address, :block] + preload: [:from_address, :to_address, :created_contract_address] ) end @@ -1052,7 +1052,7 @@ defmodule Explorer.Chain.Transaction do from( t in subquery(query), order_by: [desc: t.block_number, desc: t.index], - preload: [:from_address, :to_address, :created_contract_address, :block] + preload: [:from_address, :to_address, :created_contract_address] ) end diff --git a/apps/explorer/test/support/factory.ex b/apps/explorer/test/support/factory.ex index ac19730978..0ca6f3ee06 100644 --- a/apps/explorer/test/support/factory.ex +++ b/apps/explorer/test/support/factory.ex @@ -533,7 +533,7 @@ defmodule Explorer.Factory do %Transaction{index: nil} = transaction, # The `transaction.block` must be consensus. Non-consensus blocks can only be associated with the # `transaction_forks`. - %Block{consensus: true, hash: block_hash, number: block_number}, + %Block{consensus: true, hash: block_hash, number: block_number, timestamp: timestamp}, collated_params ) when is_list(collated_params) do @@ -555,7 +555,9 @@ defmodule Explorer.Factory do error: error, gas_used: gas_used, index: next_transaction_index, - status: status + status: status, + block_timestamp: timestamp, + block_consensus: true }) |> Repo.update!() |> Repo.preload(:block) From 0753b6c42d242bffd94559f57bdcabab5cf124e1 Mon Sep 17 00:00:00 2001 From: Viktor Baranov Date: Wed, 23 Mar 2022 13:44:11 +0300 Subject: [PATCH 08/33] lose_consensus function: remove excessive select --- CHANGELOG.md | 1 + apps/explorer/lib/explorer/chain/import/runner/blocks.ex | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f8e0db54f..addf114d8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -275,6 +275,7 @@ - [#8529](https://github.com/blockscout/blockscout/pull/8529) - Move PolygonEdge-related migration to the corresponding ecto repository - [#8504](https://github.com/blockscout/blockscout/pull/8504) - Deploy new UI through Makefile - [#8501](https://github.com/blockscout/blockscout/pull/8501) - Conceal secondary ports in docker compose setup +- [#5322](https://github.com/blockscout/blockscout/pull/5322) - DB denormalization: block consensus and timestamp in transaction table
Dependencies version bumps diff --git a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex index ffb8c47d5c..4ed9d436d9 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex @@ -401,8 +401,7 @@ defmodule Explorer.Chain.Import.Runner.Blocks do join: s in subquery(acquire_query), on: transaction.block_hash == s.hash, # we don't want to remove consensus from blocks that will be upserted - where: transaction.block_hash not in ^hashes, - select: transaction.block_hash + where: transaction.block_hash not in ^hashes ), [set: [block_consensus: false, updated_at: updated_at]], timeout: timeout From 5848a7ab216dd44209b2e1259857cbcd7a397a36 Mon Sep 17 00:00:00 2001 From: Viktor Baranov Date: Fri, 26 Aug 2022 16:10:30 +0300 Subject: [PATCH 09/33] Remove internal transactions from getLogs query --- .../api/rpc/eth_controller_test.exs | 101 ++++++++++++++---- .../api/rpc/logs_controller_test.exs | 28 ++++- apps/explorer/lib/explorer/eth_rpc.ex | 4 +- apps/explorer/lib/explorer/etherscan/logs.ex | 82 +++----------- .../test/explorer/etherscan/logs_test.exs | 64 ++++++----- 5 files changed, 160 insertions(+), 119 deletions(-) diff --git a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/eth_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/eth_controller_test.exs index f441a173b1..4b68122fd2 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/eth_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/eth_controller_test.exs @@ -76,7 +76,14 @@ defmodule BlockScoutWeb.API.RPC.EthControllerTest do block = insert(:block, number: 0) transaction = insert(:transaction, from_address: address) |> with_block(block) - insert(:log, block: block, address: address, transaction: transaction, data: "0x010101") + + insert(:log, + block: block, + block_number: block.number, + address: address, + transaction: transaction, + data: "0x010101" + ) params = params(api_params, [%{"address" => to_string(address.hash)}]) @@ -94,7 +101,15 @@ defmodule BlockScoutWeb.API.RPC.EthControllerTest do block = insert(:block, number: 0) transaction = insert(:transaction, from_address: address) |> with_block(block) - insert(:log, block: block, address: address, transaction: transaction, data: "0x010101", first_topic: "0x01") + + insert(:log, + block: block, + block_number: block.number, + address: address, + transaction: transaction, + data: "0x010101", + first_topic: "0x01" + ) params = params(api_params, [%{"address" => to_string(address.hash), "topics" => ["0x01"]}]) @@ -112,8 +127,24 @@ defmodule BlockScoutWeb.API.RPC.EthControllerTest do block = insert(:block, number: 0) transaction = insert(:transaction, from_address: address) |> with_block(block) - insert(:log, address: address, block: block, transaction: transaction, data: "0x010101", first_topic: "0x01") - insert(:log, address: address, block: block, transaction: transaction, data: "0x020202", first_topic: "0x00") + + insert(:log, + address: address, + block: block, + block_number: block.number, + transaction: transaction, + data: "0x010101", + first_topic: "0x01" + ) + + insert(:log, + address: address, + block: block, + block_number: block.number, + transaction: transaction, + data: "0x020202", + first_topic: "0x00" + ) params = params(api_params, [%{"address" => to_string(address.hash), "topics" => [["0x01", "0x00"]]}]) @@ -135,7 +166,13 @@ defmodule BlockScoutWeb.API.RPC.EthControllerTest do |> with_block(block) inserted_records = - insert_list(2000, :log, block: block, address: contract_address, transaction: transaction, first_topic: "0x01") + insert_list(2000, :log, + block: block, + block_number: block.number, + address: contract_address, + transaction: transaction, + first_topic: "0x01" + ) params = params(api_params, [%{"address" => to_string(contract_address), "topics" => [["0x01"]]}]) @@ -150,7 +187,6 @@ defmodule BlockScoutWeb.API.RPC.EthControllerTest do next_page_params = %{ "blockNumber" => Integer.to_string(transaction.block_number, 16), - "transactionIndex" => transaction.index, "logIndex" => Integer.to_string(last_log_index, 16) } @@ -193,7 +229,8 @@ defmodule BlockScoutWeb.API.RPC.EthControllerTest do data: "0x010101", first_topic: "0x01", second_topic: "0x02", - block: block + block: block, + block_number: block.number ) insert(:log, block: block, address: address, transaction: transaction, data: "0x020202", first_topic: "0x01") @@ -222,7 +259,8 @@ defmodule BlockScoutWeb.API.RPC.EthControllerTest do data: "0x010101", first_topic: "0x01", second_topic: "0x02", - block: block + block: block, + block_number: block.number ) insert(:log, @@ -231,7 +269,8 @@ defmodule BlockScoutWeb.API.RPC.EthControllerTest do data: "0x020202", first_topic: "0x01", second_topic: "0x03", - block: block + block: block, + block_number: block.number ) params = params(api_params, [%{"address" => to_string(address.hash), "topics" => ["0x01", ["0x02", "0x03"]]}]) @@ -258,13 +297,13 @@ defmodule BlockScoutWeb.API.RPC.EthControllerTest do transaction3 = insert(:transaction, from_address: address) |> with_block(block3) transaction4 = insert(:transaction, from_address: address) |> with_block(block4) - insert(:log, address: address, transaction: transaction1, data: "0x010101") + insert(:log, address: address, transaction: transaction1, data: "0x010101", block_number: block1.number) - insert(:log, address: address, transaction: transaction2, data: "0x020202") + insert(:log, address: address, transaction: transaction2, data: "0x020202", block_number: block2.number) - insert(:log, address: address, transaction: transaction3, data: "0x030303") + insert(:log, address: address, transaction: transaction3, data: "0x030303", block_number: block3.number) - insert(:log, address: address, transaction: transaction4, data: "0x040404") + insert(:log, address: address, transaction: transaction4, data: "0x040404", block_number: block4.number) params = params(api_params, [%{"address" => to_string(address.hash), "fromBlock" => 1, "toBlock" => 2}]) @@ -288,11 +327,11 @@ defmodule BlockScoutWeb.API.RPC.EthControllerTest do transaction2 = insert(:transaction, from_address: address) |> with_block(block2) transaction3 = insert(:transaction, from_address: address) |> with_block(block3) - insert(:log, address: address, transaction: transaction1, data: "0x010101") + insert(:log, address: address, transaction: transaction1, data: "0x010101", block_number: block1.number) - insert(:log, address: address, transaction: transaction2, data: "0x020202") + insert(:log, address: address, transaction: transaction2, data: "0x020202", block_number: block2.number) - insert(:log, address: address, transaction: transaction3, data: "0x030303") + insert(:log, address: address, transaction: transaction3, data: "0x030303", block_number: block3.number) params = params(api_params, [%{"address" => to_string(address.hash), "blockHash" => to_string(block2.hash)}]) @@ -316,11 +355,11 @@ defmodule BlockScoutWeb.API.RPC.EthControllerTest do transaction2 = insert(:transaction, from_address: address) |> with_block(block2) transaction3 = insert(:transaction, from_address: address) |> with_block(block3) - insert(:log, address: address, transaction: transaction1, data: "0x010101") + insert(:log, address: address, transaction: transaction1, data: "0x010101", block_number: block1.number) - insert(:log, address: address, transaction: transaction2, data: "0x020202") + insert(:log, address: address, transaction: transaction2, data: "0x020202", block_number: block2.number) - insert(:log, address: address, transaction: transaction3, data: "0x030303") + insert(:log, address: address, transaction: transaction3, data: "0x030303", block_number: block3.number) params = params(api_params, [%{"address" => to_string(address.hash), "fromBlock" => "earliest", "toBlock" => "earliest"}]) @@ -345,11 +384,29 @@ defmodule BlockScoutWeb.API.RPC.EthControllerTest do transaction2 = insert(:transaction, from_address: address) |> with_block(block2) transaction3 = insert(:transaction, from_address: address) |> with_block(block3) - insert(:log, block: block1, address: address, transaction: transaction1, data: "0x010101") + insert(:log, + block: block1, + block_number: block1.number, + address: address, + transaction: transaction1, + data: "0x010101" + ) - insert(:log, block: block2, address: address, transaction: transaction2, data: "0x020202") + insert(:log, + block: block2, + block_number: block2.number, + address: address, + transaction: transaction2, + data: "0x020202" + ) - insert(:log, block: block3, address: address, transaction: transaction3, data: "0x030303") + insert(:log, + block: block3, + block_number: block3.number, + address: address, + transaction: transaction3, + data: "0x030303" + ) changeset = Ecto.Changeset.change(block3, %{consensus: false}) diff --git a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/logs_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/logs_controller_test.exs index ec4337eecd..66fc27dc71 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/logs_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/logs_controller_test.exs @@ -280,7 +280,7 @@ defmodule BlockScoutWeb.API.RPC.LogsControllerTest do |> insert(to_address: contract_address) |> with_block() - log = insert(:log, address: contract_address, transaction: transaction) + log = insert(:log, address: contract_address, transaction: transaction, block_number: transaction.block_number) params = %{ "module" => "logs", @@ -334,8 +334,17 @@ defmodule BlockScoutWeb.API.RPC.LogsControllerTest do |> insert(to_address: contract_address) |> with_block(second_block) - insert(:log, address: contract_address, transaction: transaction_block1) - insert(:log, address: contract_address, transaction: transaction_block2) + insert(:log, + address: contract_address, + transaction: transaction_block1, + block_number: transaction_block1.block_number + ) + + insert(:log, + address: contract_address, + transaction: transaction_block2, + block_number: transaction_block2.block_number + ) params = %{ "module" => "logs", @@ -378,8 +387,17 @@ defmodule BlockScoutWeb.API.RPC.LogsControllerTest do |> insert(to_address: contract_address) |> with_block(second_block) - insert(:log, address: contract_address, transaction: transaction_block1) - insert(:log, address: contract_address, transaction: transaction_block2) + insert(:log, + address: contract_address, + transaction: transaction_block1, + block_number: transaction_block1.block_number + ) + + insert(:log, + address: contract_address, + transaction: transaction_block2, + block_number: transaction_block2.block_number + ) params = %{ "module" => "logs", diff --git a/apps/explorer/lib/explorer/eth_rpc.ex b/apps/explorer/lib/explorer/eth_rpc.ex index 889fdabbba..0e5b493a9a 100644 --- a/apps/explorer/lib/explorer/eth_rpc.ex +++ b/apps/explorer/lib/explorer/eth_rpc.ex @@ -54,13 +54,13 @@ defmodule Explorer.EthRPC do action: :eth_get_logs, notes: """ Will never return more than 1000 log entries.\n - For this reason, you can use pagination options to request the next page. Pagination options params: {"logIndex": "3D", "blockNumber": "6423AC", "transactionIndex": 53} which include parameters from the last log received from the previous request. These three parameters are required for pagination. + For this reason, you can use pagination options to request the next page. Pagination options params: {"logIndex": "3D", "blockNumber": "6423AC"} which include parameters from the last log received from the previous request. These three parameters are required for pagination. """, example: """ {"id": 0, "jsonrpc": "2.0", "method": "eth_getLogs", "params": [ {"address": "0xc78Be425090Dbd437532594D12267C5934Cc6c6f", - "paging_options": {"logIndex": "3D", "blockNumber": "6423AC", "transactionIndex": 53}, + "paging_options": {"logIndex": "3D", "blockNumber": "6423AC"}, "fromBlock": "earliest", "toBlock": "latest", "topics": ["0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"]}]} diff --git a/apps/explorer/lib/explorer/etherscan/logs.ex b/apps/explorer/lib/explorer/etherscan/logs.ex index 4d4e4111f9..70c4dedaf2 100644 --- a/apps/explorer/lib/explorer/etherscan/logs.ex +++ b/apps/explorer/lib/explorer/etherscan/logs.ex @@ -5,10 +5,10 @@ defmodule Explorer.Etherscan.Logs do """ - import Ecto.Query, only: [from: 2, where: 3, subquery: 1, order_by: 3, union: 2] + import Ecto.Query, only: [from: 2, limit: 2, where: 3, subquery: 1, order_by: 3] - alias Explorer.{Chain, Repo} - alias Explorer.Chain.{InternalTransaction, Log, Transaction} + alias Explorer.Repo + alias Explorer.Chain.{Log, Transaction} @base_filter %{ from_block: nil, @@ -38,7 +38,7 @@ defmodule Explorer.Etherscan.Logs do :type ] - @default_paging_options %{block_number: nil, transaction_index: nil, log_index: nil} + @default_paging_options %{block_number: nil, log_index: nil} @doc """ Gets a list of logs that meet the criteria in a given filter map. @@ -76,38 +76,20 @@ defmodule Explorer.Etherscan.Logs do paging_options = if is_nil(paging_options), do: @default_paging_options, else: paging_options prepared_filter = Map.merge(@base_filter, filter) - logs_query = where_topic_match(Log, prepared_filter) - - query_to_address_hash_wrapped = - logs_query - |> internal_transaction_query(:to_address_hash, prepared_filter, address_hash) - |> Chain.wrapped_union_subquery() - - query_from_address_hash_wrapped = - logs_query - |> internal_transaction_query(:from_address_hash, prepared_filter, address_hash) - |> Chain.wrapped_union_subquery() - - query_created_contract_address_hash_wrapped = - logs_query - |> internal_transaction_query(:created_contract_address_hash, prepared_filter, address_hash) - |> Chain.wrapped_union_subquery() - - internal_transaction_log_query = - query_to_address_hash_wrapped - |> union(^query_from_address_hash_wrapped) - |> union(^query_created_contract_address_hash_wrapped) + logs_query = + Log + |> where_topic_match(prepared_filter) + |> where([log], log.address_hash == ^address_hash) + |> where([log], log.block_number >= ^prepared_filter.from_block) + |> where([log], log.block_number <= ^prepared_filter.to_block) + |> limit(1000) + |> order_by([log], asc: log.block_number, asc: log.index) + |> page_logs(paging_options) all_transaction_logs_query = - from(transaction in Transaction, - join: log in ^logs_query, + from(log in subquery(logs_query), + join: transaction in Transaction, on: log.transaction_hash == transaction.hash, - where: transaction.block_number >= ^prepared_filter.from_block, - where: transaction.block_number <= ^prepared_filter.to_block, - where: - transaction.to_address_hash == ^address_hash or - transaction.from_address_hash == ^address_hash or - transaction.created_contract_address_hash == ^address_hash, select: map(log, ^@log_fields), select_merge: %{ gas_price: transaction.gas_price, @@ -117,20 +99,14 @@ defmodule Explorer.Etherscan.Logs do block_number: transaction.block_number, block_timestamp: transaction.block_timestamp, block_consensus: transaction.block_consensus - }, - union: ^internal_transaction_log_query + } ) query_with_blocks = from(log_transaction_data in subquery(all_transaction_logs_query), where: log_transaction_data.address_hash == ^address_hash, order_by: log_transaction_data.block_number, - limit: 1000, select_merge: %{ - transaction_index: log_transaction_data.transaction_index, - block_hash: log_transaction_data.block_hash, - block_number: log_transaction_data.block_number, - block_timestamp: log_transaction_data.block_timestamp, block_consensus: log_transaction_data.block_consensus } ) @@ -145,8 +121,6 @@ defmodule Explorer.Etherscan.Logs do end query_with_consensus - |> order_by([log], asc: log.index) - |> page_logs(paging_options) |> Repo.replica().all() end @@ -258,28 +232,4 @@ defmodule Explorer.Etherscan.Logs do where: data.index > ^log_index and data.block_number >= ^block_number ) end - - defp internal_transaction_query(logs_query, direction, prepared_filter, address_hash) do - query = - from(internal_transaction in InternalTransaction.where_nonpending_block(), - join: transaction in assoc(internal_transaction, :transaction), - join: log in ^logs_query, - on: log.transaction_hash == internal_transaction.transaction_hash, - where: internal_transaction.block_number >= ^prepared_filter.from_block, - where: internal_transaction.block_number <= ^prepared_filter.to_block, - select: - merge(map(log, ^@log_fields), %{ - gas_price: transaction.gas_price, - gas_used: transaction.gas_used, - transaction_index: transaction.index, - block_hash: transaction.block_hash, - block_number: internal_transaction.block_number, - block_timestamp: transaction.block_timestamp, - block_consensus: transaction.block_consensus - }) - ) - - query - |> InternalTransaction.where_address_fields_match(address_hash, direction) - end end diff --git a/apps/explorer/test/explorer/etherscan/logs_test.exs b/apps/explorer/test/explorer/etherscan/logs_test.exs index 81b04e01b3..5da949541c 100644 --- a/apps/explorer/test/explorer/etherscan/logs_test.exs +++ b/apps/explorer/test/explorer/etherscan/logs_test.exs @@ -46,7 +46,7 @@ defmodule Explorer.Etherscan.LogsTest do |> insert(to_address: contract_address, block_timestamp: block.timestamp) |> with_block(block) - log = insert(:log, address: contract_address, transaction: transaction) + log = insert(:log, address: contract_address, block_number: block.number, transaction: transaction) filter = %{ from_block: block.number, @@ -80,7 +80,7 @@ defmodule Explorer.Etherscan.LogsTest do |> insert(to_address: contract_address) |> with_block() - insert_list(2, :log, address: contract_address, transaction: transaction) + insert_list(2, :log, address: contract_address, transaction: transaction, block_number: block.number) filter = %{ from_block: block.number, @@ -111,8 +111,8 @@ defmodule Explorer.Etherscan.LogsTest do |> insert(to_address: contract_address) |> with_block(second_block) - insert(:log, address: contract_address, transaction: transaction_block1) - insert(:log, address: contract_address, transaction: transaction_block2) + insert(:log, address: contract_address, transaction: transaction_block1, block_number: first_block.number) + insert(:log, address: contract_address, transaction: transaction_block2, block_number: second_block.number) filter = %{ from_block: second_block.number, @@ -144,8 +144,8 @@ defmodule Explorer.Etherscan.LogsTest do |> insert(to_address: contract_address) |> with_block(second_block) - insert(:log, address: contract_address, transaction: transaction_block1) - insert(:log, address: contract_address, transaction: transaction_block2) + insert(:log, address: contract_address, transaction: transaction_block1, block_number: first_block.number) + insert(:log, address: contract_address, transaction: transaction_block2, block_number: second_block.number) filter = %{ from_block: first_block.number, @@ -168,7 +168,8 @@ defmodule Explorer.Etherscan.LogsTest do |> insert(to_address: contract_address) |> with_block() - inserted_records = insert_list(2000, :log, address: contract_address, transaction: transaction) + inserted_records = + insert_list(2000, :log, address: contract_address, transaction: transaction, block_number: block.number) filter = %{ from_block: block.number, @@ -184,7 +185,6 @@ defmodule Explorer.Etherscan.LogsTest do next_page_params = %{ log_index: last_record.index, - transaction_index: last_record.transaction_index, block_number: transaction.block_number } @@ -327,13 +327,15 @@ defmodule Explorer.Etherscan.LogsTest do log1_details = [ address: contract_address, transaction: transaction, - first_topic: "some first topic" + first_topic: "some first topic", + block_number: block.number ] log2_details = [ address: contract_address, transaction: transaction, - first_topic: "some OTHER first topic" + first_topic: "some OTHER first topic", + block_number: block.number ] _log1 = insert(:log, log1_details) @@ -365,14 +367,16 @@ defmodule Explorer.Etherscan.LogsTest do address: contract_address, transaction: transaction, first_topic: "some first topic", - second_topic: "some second topic" + second_topic: "some second topic", + block_number: block.number ] log2_details = [ address: contract_address, transaction: transaction, first_topic: "some OTHER first topic", - second_topic: "some OTHER second topic" + second_topic: "some OTHER second topic", + block_number: block.number ] _log1 = insert(:log, log1_details) @@ -407,7 +411,8 @@ defmodule Explorer.Etherscan.LogsTest do transaction: transaction, first_topic: "some first topic", second_topic: "some second topic", - third_topic: "some third topic" + third_topic: "some third topic", + block_number: block.number ] log2_details = [ @@ -415,7 +420,8 @@ defmodule Explorer.Etherscan.LogsTest do transaction: transaction, first_topic: "some OTHER first topic", second_topic: "some OTHER second topic", - third_topic: "some OTHER third topic" + third_topic: "some OTHER third topic", + block_number: block.number ] log3_details = [ @@ -423,7 +429,8 @@ defmodule Explorer.Etherscan.LogsTest do transaction: transaction, first_topic: "some ALT first topic", second_topic: "some ALT second topic", - third_topic: "some ALT third topic" + third_topic: "some ALT third topic", + block_number: block.number ] _log1 = insert(:log, log1_details) @@ -464,7 +471,8 @@ defmodule Explorer.Etherscan.LogsTest do transaction: transaction, first_topic: "some first topic", second_topic: "some second topic", - third_topic: "some third topic" + third_topic: "some third topic", + block_number: block.number ] log2_details = [ @@ -472,7 +480,8 @@ defmodule Explorer.Etherscan.LogsTest do transaction: transaction, first_topic: "some OTHER first topic", second_topic: "some OTHER second topic", - third_topic: "some OTHER third topic" + third_topic: "some OTHER third topic", + block_number: block.number ] log3_details = [ @@ -480,7 +489,8 @@ defmodule Explorer.Etherscan.LogsTest do transaction: transaction, first_topic: "some ALT first topic", second_topic: "some ALT second topic", - third_topic: "some ALT third topic" + third_topic: "some ALT third topic", + block_number: block.number ] log1 = insert(:log, log1_details) @@ -521,7 +531,8 @@ defmodule Explorer.Etherscan.LogsTest do transaction: transaction, first_topic: "some topic", second_topic: "some second topic", - third_topic: "some third topic" + third_topic: "some third topic", + block_number: block.number ] log2_details = [ @@ -529,7 +540,8 @@ defmodule Explorer.Etherscan.LogsTest do transaction: transaction, first_topic: "some topic", second_topic: "some OTHER second topic", - third_topic: "some third topic" + third_topic: "some third topic", + block_number: block.number ] log3_details = [ @@ -537,7 +549,8 @@ defmodule Explorer.Etherscan.LogsTest do transaction: transaction, first_topic: "some topic", second_topic: "some second topic", - third_topic: "some third topic" + third_topic: "some third topic", + block_number: block.number ] log1 = insert(:log, log1_details) @@ -577,7 +590,8 @@ defmodule Explorer.Etherscan.LogsTest do address: contract_address, transaction: transaction, first_topic: "some topic", - second_topic: "some second topic" + second_topic: "some second topic", + block_number: block.number ] log2_details = [ @@ -586,7 +600,8 @@ defmodule Explorer.Etherscan.LogsTest do first_topic: "some OTHER topic", second_topic: "some OTHER second topic", third_topic: "some OTHER third topic", - fourth_topic: "some fourth topic" + fourth_topic: "some fourth topic", + block_number: block.number ] log3_details = [ @@ -595,7 +610,8 @@ defmodule Explorer.Etherscan.LogsTest do first_topic: "some topic", second_topic: "some second topic", third_topic: "some third topic", - fourth_topic: "some fourth topic" + fourth_topic: "some fourth topic", + block_number: block.number ] log1 = insert(:log, log1_details) From e1c6f44c17ca8ad8c62fd39840eef93c52d4ba6c Mon Sep 17 00:00:00 2001 From: Viktor Baranov Date: Mon, 29 Aug 2022 16:41:22 +0300 Subject: [PATCH 10/33] Handle nil timestamp in datetime_to_hex function --- .../lib/block_scout_web/views/api/rpc/logs_view.ex | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/block_scout_web/lib/block_scout_web/views/api/rpc/logs_view.ex b/apps/block_scout_web/lib/block_scout_web/views/api/rpc/logs_view.ex index 6f2933d7c6..f52ab8fadc 100644 --- a/apps/block_scout_web/lib/block_scout_web/views/api/rpc/logs_view.ex +++ b/apps/block_scout_web/lib/block_scout_web/views/api/rpc/logs_view.ex @@ -44,6 +44,8 @@ defmodule BlockScoutWeb.API.RPC.LogsView do |> integer_to_hex() end + defp datetime_to_hex(nil), do: nil + defp datetime_to_hex(datetime) do datetime |> DateTime.to_unix() From fd13a8c2698aa6f908045d2303f5b1be1705cec8 Mon Sep 17 00:00:00 2001 From: Qwerty5Uiop Date: Thu, 2 Nov 2023 22:52:52 +0600 Subject: [PATCH 11/33] Denormalization migration + dynamic logic support --- CHANGELOG.md | 2 +- .../address_token_transfer_controller.ex | 3 +- .../address_transaction_controller.ex | 3 +- .../api/rpc/transaction_controller.ex | 4 +- .../api/v2/transaction_controller.ex | 1 - .../recent_transactions_controller.ex | 28 +- .../controllers/transaction_controller.ex | 12 +- ...saction_internal_transaction_controller.ex | 28 +- .../lib/block_scout_web/notifier.ex | 6 +- .../tokens/transfer/_token_transfer.html.eex | 2 +- .../views/api/rpc/transaction_view.ex | 3 +- .../views/api/v2/transaction_view.ex | 3 +- .../block_scout_web/views/transaction_view.ex | 4 +- apps/block_scout_web/priv/gettext/default.pot | 48 +-- .../priv/gettext/en/LC_MESSAGES/default.po | 48 +-- .../channels/websocket_v2_test.exs | 2 + .../api/v2/search_controller_test.exs | 2 +- apps/explorer/config/config.exs | 2 + apps/explorer/config/runtime/test.exs | 2 + apps/explorer/lib/explorer/application.ex | 5 +- apps/explorer/lib/explorer/chain.ex | 23 +- .../chain/cache/background_migrations.ex | 23 ++ .../address_token_transfer_csv_exporter.ex | 4 +- .../address_transaction_csv_exporter.ex | 17 +- .../explorer/chain/denormalization_helper.ex | 44 +++ .../chain/import/runner/transactions.ex | 4 +- apps/explorer/lib/explorer/chain/search.ex | 95 ++++-- .../lib/explorer/chain/token_transfer.ex | 8 +- .../lib/explorer/chain/transaction.ex | 12 +- .../chain/transaction/history/historian.ex | 55 +++- apps/explorer/lib/explorer/etherscan.ex | 238 +++++++++----- apps/explorer/lib/explorer/etherscan/logs.ex | 299 +++++++++++++----- .../transactions_denormalization_migrator.ex | 91 ++++++ ...902_add_consensus_to_transaction_table.exs | 8 - ...d_block_timestamp_to_transaction_table.exs | 7 - ...nsactions_denormalization_migrator_test.ex | 37 +++ apps/explorer/test/support/factory.ex | 6 +- cspell.json | 2 + 38 files changed, 849 insertions(+), 332 deletions(-) create mode 100644 apps/explorer/lib/explorer/chain/cache/background_migrations.ex create mode 100644 apps/explorer/lib/explorer/chain/denormalization_helper.ex create mode 100644 apps/explorer/lib/explorer/transactions_denormalization_migrator.ex create mode 100644 apps/explorer/test/explorer/transactions_denormalization_migrator_test.ex diff --git a/CHANGELOG.md b/CHANGELOG.md index addf114d8d..d07488b208 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -101,6 +101,7 @@ - [#8956](https://github.com/blockscout/blockscout/pull/8956) - Refine docker-compose config structure - [#8911](https://github.com/blockscout/blockscout/pull/8911) - Set client_connection_check_interval for main Postgres DB in docker-compose setup +- [#5322](https://github.com/blockscout/blockscout/pull/5322) - DB denormalization: block consensus and timestamp in transaction table
Dependencies version bumps @@ -275,7 +276,6 @@ - [#8529](https://github.com/blockscout/blockscout/pull/8529) - Move PolygonEdge-related migration to the corresponding ecto repository - [#8504](https://github.com/blockscout/blockscout/pull/8504) - Deploy new UI through Makefile - [#8501](https://github.com/blockscout/blockscout/pull/8501) - Conceal secondary ports in docker compose setup -- [#5322](https://github.com/blockscout/blockscout/pull/5322) - DB denormalization: block consensus and timestamp in transaction table
Dependencies version bumps diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/address_token_transfer_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/address_token_transfer_controller.ex index 743b063e74..94f87de093 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/address_token_transfer_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/address_token_transfer_controller.ex @@ -6,7 +6,7 @@ defmodule BlockScoutWeb.AddressTokenTransferController do alias BlockScoutWeb.{AccessHelper, Controller, TransactionView} alias Explorer.{Chain, Market} - alias Explorer.Chain.Address + alias Explorer.Chain.{Address, DenormalizationHelper} alias Indexer.Fetcher.CoinBalanceOnDemand alias Phoenix.View @@ -140,6 +140,7 @@ defmodule BlockScoutWeb.AddressTokenTransferController do {:ok, false} <- AccessHelper.restricted_access?(address_hash_string, params) do options = @transaction_necessity_by_association + |> DenormalizationHelper.extend_block_necessity(:required) |> Keyword.merge(paging_options(params)) |> Keyword.merge(current_filter(params)) diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/address_transaction_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/address_transaction_controller.ex index 28543dfe2b..05ef0c98d7 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/address_transaction_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/address_transaction_controller.ex @@ -20,7 +20,7 @@ defmodule BlockScoutWeb.AddressTransactionController do AddressTransactionCsvExporter } - alias Explorer.Chain.{Transaction, Wei} + alias Explorer.Chain.{DenormalizationHelper, Transaction, Wei} alias Indexer.Fetcher.CoinBalanceOnDemand alias Phoenix.View @@ -49,6 +49,7 @@ defmodule BlockScoutWeb.AddressTransactionController do {:ok, false} <- AccessHelper.restricted_access?(address_hash_string, params) do options = @transaction_necessity_by_association + |> DenormalizationHelper.extend_block_necessity(:optional) |> Keyword.merge(paging_options(params)) |> Keyword.merge(current_filter(params)) diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/api/rpc/transaction_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/api/rpc/transaction_controller.ex index 28014a2eef..fbea15ca76 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/api/rpc/transaction_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/api/rpc/transaction_controller.ex @@ -4,7 +4,7 @@ defmodule BlockScoutWeb.API.RPC.TransactionController do import BlockScoutWeb.Chain, only: [paging_options: 1, next_page_params: 3, split_list_by_page: 1] alias Explorer.Chain - alias Explorer.Chain.Transaction + alias Explorer.Chain.{DenormalizationHelper, Transaction} @api_true [api?: true] @@ -75,7 +75,7 @@ defmodule BlockScoutWeb.API.RPC.TransactionController do end defp transaction_from_hash(transaction_hash) do - case Chain.hash_to_transaction(transaction_hash) do + case Chain.hash_to_transaction(transaction_hash, DenormalizationHelper.extend_block_necessity([], :required)) do {:error, :not_found} -> {:transaction, :error} {:ok, transaction} -> {:transaction, {:ok, transaction}} end diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/transaction_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/transaction_controller.ex index 993d2ac68a..a1e5134f4e 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/transaction_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/transaction_controller.ex @@ -62,7 +62,6 @@ defmodule BlockScoutWeb.API.V2.TransactionController do [created_contract_address: :names] => :optional, [from_address: :names] => :optional, [to_address: :names] => :optional, - [transaction: :block] => :optional, [created_contract_address: :smart_contract] => :optional, [from_address: :smart_contract] => :optional, [to_address: :smart_contract] => :optional diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/recent_transactions_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/recent_transactions_controller.ex index ac7e39b822..0863579eab 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/recent_transactions_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/recent_transactions_controller.ex @@ -4,7 +4,7 @@ defmodule BlockScoutWeb.RecentTransactionsController do import Explorer.Chain.SmartContract, only: [burn_address_hash_string: 0] alias Explorer.{Chain, PagingOptions} - alias Explorer.Chain.Hash + alias Explorer.Chain.{DenormalizationHelper, Hash} alias Phoenix.View {:ok, burn_address_hash} = Chain.string_to_address_hash(burn_address_hash_string()) @@ -13,16 +13,22 @@ defmodule BlockScoutWeb.RecentTransactionsController do def index(conn, _params) do if ajax?(conn) do recent_transactions = - Chain.recent_collated_transactions(true, - necessity_by_association: %{ - [created_contract_address: :names] => :optional, - [from_address: :names] => :optional, - [to_address: :names] => :optional, - [created_contract_address: :smart_contract] => :optional, - [from_address: :smart_contract] => :optional, - [to_address: :smart_contract] => :optional - }, - paging_options: %PagingOptions{page_size: 5} + Chain.recent_collated_transactions( + true, + DenormalizationHelper.extend_block_necessity( + [ + necessity_by_association: %{ + [created_contract_address: :names] => :optional, + [from_address: :names] => :optional, + [to_address: :names] => :optional, + [created_contract_address: :smart_contract] => :optional, + [from_address: :smart_contract] => :optional, + [to_address: :smart_contract] => :optional + }, + paging_options: %PagingOptions{page_size: 5} + ], + :required + ) ) transactions = diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/transaction_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/transaction_controller.ex index 7c0bd30526..a31baf1ea9 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/transaction_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/transaction_controller.ex @@ -26,6 +26,7 @@ defmodule BlockScoutWeb.TransactionController do alias Explorer.{Chain, Market} alias Explorer.Chain.Cache.Transaction, as: TransactionCache + alias Explorer.Chain.DenormalizationHelper alias Phoenix.View @necessity_by_association %{ @@ -54,6 +55,7 @@ defmodule BlockScoutWeb.TransactionController do def index(conn, %{"type" => "JSON"} = params) do options = @default_options + |> DenormalizationHelper.extend_block_necessity(:required) |> Keyword.merge(paging_options(params)) full_options = @@ -151,10 +153,7 @@ defmodule BlockScoutWeb.TransactionController do :ok <- Chain.check_transaction_exists(transaction_hash) do if Chain.transaction_has_token_transfers?(transaction_hash) do with {:ok, transaction} <- - Chain.hash_to_transaction( - transaction_hash, - necessity_by_association: @necessity_by_association - ), + Chain.hash_to_transaction(transaction_hash, necessity_by_association: @necessity_by_association), {:ok, false} <- AccessHelper.restricted_access?(to_string(transaction.from_address_hash), params), {:ok, false} <- AccessHelper.restricted_access?(to_string(transaction.to_address_hash), params) do render( @@ -189,10 +188,7 @@ defmodule BlockScoutWeb.TransactionController do end else with {:ok, transaction} <- - Chain.hash_to_transaction( - transaction_hash, - necessity_by_association: @necessity_by_association - ), + Chain.hash_to_transaction(transaction_hash, necessity_by_association: @necessity_by_association), {:ok, false} <- AccessHelper.restricted_access?(to_string(transaction.from_address_hash), params), {:ok, false} <- AccessHelper.restricted_access?(to_string(transaction.to_address_hash), params) do render( diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/transaction_internal_transaction_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/transaction_internal_transaction_controller.ex index 4109f473e1..5ea1e44721 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/transaction_internal_transaction_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/transaction_internal_transaction_controller.ex @@ -8,6 +8,7 @@ defmodule BlockScoutWeb.TransactionInternalTransactionController do alias BlockScoutWeb.{AccessHelper, Controller, InternalTransactionView, TransactionController} alias Explorer.{Chain, Market} + alias Explorer.Chain.DenormalizationHelper alias Phoenix.View def index(conn, %{"transaction_id" => transaction_hash_string, "type" => "JSON"} = params) do @@ -17,20 +18,19 @@ defmodule BlockScoutWeb.TransactionInternalTransactionController do {:ok, false} <- AccessHelper.restricted_access?(to_string(transaction.from_address_hash), params), {:ok, false} <- AccessHelper.restricted_access?(to_string(transaction.to_address_hash), params) do full_options = - Keyword.merge( - [ - necessity_by_association: %{ - [created_contract_address: :names] => :optional, - [from_address: :names] => :optional, - [to_address: :names] => :optional, - [created_contract_address: :smart_contract] => :optional, - [from_address: :smart_contract] => :optional, - [to_address: :smart_contract] => :optional, - :transaction => :optional - } - ], - paging_options(params) - ) + [ + necessity_by_association: %{ + [created_contract_address: :names] => :optional, + [from_address: :names] => :optional, + [to_address: :names] => :optional, + [created_contract_address: :smart_contract] => :optional, + [from_address: :smart_contract] => :optional, + [to_address: :smart_contract] => :optional, + :transaction => :optional + } + ] + |> DenormalizationHelper.extend_transaction_block_necessity(:optional) + |> Keyword.merge(paging_options(params)) internal_transactions_plus_one = Chain.transaction_to_internal_transactions(transaction_hash, full_options) diff --git a/apps/block_scout_web/lib/block_scout_web/notifier.ex b/apps/block_scout_web/lib/block_scout_web/notifier.ex index 1de89cee8b..3a70cd3165 100644 --- a/apps/block_scout_web/lib/block_scout_web/notifier.ex +++ b/apps/block_scout_web/lib/block_scout_web/notifier.ex @@ -20,7 +20,7 @@ defmodule BlockScoutWeb.Notifier do alias Explorer.{Chain, Market, Repo} alias Explorer.Chain.Address.Counters - alias Explorer.Chain.{Address, InternalTransaction, Transaction} + alias Explorer.Chain.{Address, DenormalizationHelper, InternalTransaction, Transaction} alias Explorer.Chain.Supply.RSK alias Explorer.Chain.Transaction.History.TransactionStats alias Explorer.Counters.{AverageBlockTime, Helper} @@ -171,7 +171,9 @@ defmodule BlockScoutWeb.Notifier do all_token_transfers |> Enum.map( &(&1 - |> Repo.preload([:from_address, :to_address, :token, transaction: :block])) + |> Repo.preload( + DenormalizationHelper.extend_transaction_preload([:from_address, :to_address, :token, :transaction]) + )) ) transfers_by_token = Enum.group_by(all_token_transfers_full, fn tt -> to_string(tt.token_contract_address_hash) end) diff --git a/apps/block_scout_web/lib/block_scout_web/templates/tokens/transfer/_token_transfer.html.eex b/apps/block_scout_web/lib/block_scout_web/templates/tokens/transfer/_token_transfer.html.eex index 261e209c8d..73b6bbf5af 100644 --- a/apps/block_scout_web/lib/block_scout_web/templates/tokens/transfer/_token_transfer.html.eex +++ b/apps/block_scout_web/lib/block_scout_web/templates/tokens/transfer/_token_transfer.html.eex @@ -44,7 +44,7 @@ to: block_path(BlockScoutWeb.Endpoint, :show, @token_transfer.block_number) ) %> - + diff --git a/apps/block_scout_web/lib/block_scout_web/views/api/rpc/transaction_view.ex b/apps/block_scout_web/lib/block_scout_web/views/api/rpc/transaction_view.ex index 2e7713fa50..4a18643aa3 100644 --- a/apps/block_scout_web/lib/block_scout_web/views/api/rpc/transaction_view.ex +++ b/apps/block_scout_web/lib/block_scout_web/views/api/rpc/transaction_view.ex @@ -2,6 +2,7 @@ defmodule BlockScoutWeb.API.RPC.TransactionView do use BlockScoutWeb, :view alias BlockScoutWeb.API.RPC.RPCView + alias Explorer.Chain.Transaction def render("gettxinfo.json", %{ transaction: transaction, @@ -58,7 +59,7 @@ defmodule BlockScoutWeb.API.RPC.TransactionView do defp prepare_transaction(transaction, block_height, logs, next_page_params) do %{ "hash" => "#{transaction.hash}", - "timeStamp" => "#{DateTime.to_unix(transaction.block_timestamp)}", + "timeStamp" => "#{DateTime.to_unix(Transaction.block_timestamp(transaction))}", "blockNumber" => "#{transaction.block_number}", "confirmations" => "#{block_height - transaction.block_number}", "success" => if(transaction.status == :ok, do: true, else: false), diff --git a/apps/block_scout_web/lib/block_scout_web/views/api/v2/transaction_view.ex b/apps/block_scout_web/lib/block_scout_web/views/api/v2/transaction_view.ex index 28bc4042c4..44832a5127 100644 --- a/apps/block_scout_web/lib/block_scout_web/views/api/v2/transaction_view.ex +++ b/apps/block_scout_web/lib/block_scout_web/views/api/v2/transaction_view.ex @@ -375,7 +375,7 @@ defmodule BlockScoutWeb.API.V2.TransactionView do "result" => status, "status" => transaction.status, "block" => transaction.block_number, - "timestamp" => block_timestamp(transaction.block), + "timestamp" => block_timestamp(transaction), "from" => Helper.address_with_info( single_tx? && conn, @@ -833,6 +833,7 @@ defmodule BlockScoutWeb.API.V2.TransactionView do end end + defp block_timestamp(%Transaction{block_timestamp: block_ts}) when not is_nil(block_ts), do: block_ts defp block_timestamp(%Transaction{block: %Block{} = block}), do: block.timestamp defp block_timestamp(%Block{} = block), do: block.timestamp defp block_timestamp(_), do: nil diff --git a/apps/block_scout_web/lib/block_scout_web/views/transaction_view.ex b/apps/block_scout_web/lib/block_scout_web/views/transaction_view.ex index 3112ff3091..ae493f3913 100644 --- a/apps/block_scout_web/lib/block_scout_web/views/transaction_view.ex +++ b/apps/block_scout_web/lib/block_scout_web/views/transaction_view.ex @@ -38,9 +38,7 @@ defmodule BlockScoutWeb.TransactionView do def block_number(%Reward{block: block}), do: [view_module: BlockView, partial: "_link.html", block: block] - def block_timestamp(%Transaction{block_number: nil, inserted_at: time}), do: time - def block_timestamp(%Transaction{block_timestamp: time}), do: time - def block_timestamp(%Transaction{block: %Block{timestamp: time}}), do: time + def block_timestamp(%Transaction{} = transaction), do: Transaction.block_timestamp(transaction) def block_timestamp(%Reward{block: %Block{timestamp: time}}), do: time def value_transfer?(%Transaction{input: %{bytes: bytes}}) when bytes in [<<>>, nil] do diff --git a/apps/block_scout_web/priv/gettext/default.pot b/apps/block_scout_web/priv/gettext/default.pot index 096df745c2..224aa1df77 100644 --- a/apps/block_scout_web/priv/gettext/default.pot +++ b/apps/block_scout_web/priv/gettext/default.pot @@ -68,7 +68,7 @@ msgstr "" msgid "%{subnetwork} Explorer - BlockScout" msgstr "" -#: lib/block_scout_web/views/transaction_view.ex:369 +#: lib/block_scout_web/views/transaction_view.ex:371 #, elixir-autogen, elixir-format msgid "(Awaiting internal transactions for status)" msgstr "" @@ -698,7 +698,7 @@ msgstr "" msgid "Compiler version" msgstr "" -#: lib/block_scout_web/views/transaction_view.ex:362 +#: lib/block_scout_web/views/transaction_view.ex:364 #, elixir-autogen, elixir-format msgid "Confirmed" msgstr "" @@ -783,12 +783,12 @@ msgstr "" msgid "Contract Address Pending" msgstr "" -#: lib/block_scout_web/views/transaction_view.ex:478 +#: lib/block_scout_web/views/transaction_view.ex:480 #, elixir-autogen, elixir-format msgid "Contract Call" msgstr "" -#: lib/block_scout_web/views/transaction_view.ex:475 +#: lib/block_scout_web/views/transaction_view.ex:477 #, elixir-autogen, elixir-format msgid "Contract Creation" msgstr "" @@ -1186,12 +1186,12 @@ msgstr "" msgid "EIP-1167" msgstr "" -#: lib/block_scout_web/views/transaction_view.ex:220 +#: lib/block_scout_web/views/transaction_view.ex:222 #, elixir-autogen, elixir-format msgid "ERC-1155 " msgstr "" -#: lib/block_scout_web/views/transaction_view.ex:218 +#: lib/block_scout_web/views/transaction_view.ex:220 #, elixir-autogen, elixir-format msgid "ERC-20 " msgstr "" @@ -1201,7 +1201,7 @@ msgstr "" msgid "ERC-20 tokens (beta)" msgstr "" -#: lib/block_scout_web/views/transaction_view.ex:219 +#: lib/block_scout_web/views/transaction_view.ex:221 #, elixir-autogen, elixir-format msgid "ERC-721 " msgstr "" @@ -1292,12 +1292,12 @@ msgstr "" msgid "Error trying to fetch balances." msgstr "" -#: lib/block_scout_web/views/transaction_view.ex:373 +#: lib/block_scout_web/views/transaction_view.ex:375 #, elixir-autogen, elixir-format msgid "Error: %{reason}" msgstr "" -#: lib/block_scout_web/views/transaction_view.ex:371 +#: lib/block_scout_web/views/transaction_view.ex:373 #, elixir-autogen, elixir-format msgid "Error: (Awaiting internal transactions for reason)" msgstr "" @@ -1602,7 +1602,7 @@ msgstr "" #: lib/block_scout_web/templates/transaction/_tabs.html.eex:11 #: lib/block_scout_web/templates/transaction_internal_transaction/index.html.eex:6 #: lib/block_scout_web/views/address_view.ex:376 -#: lib/block_scout_web/views/transaction_view.ex:533 +#: lib/block_scout_web/views/transaction_view.ex:535 #, elixir-autogen, elixir-format msgid "Internal Transactions" msgstr "" @@ -1719,7 +1719,7 @@ msgstr "" #: lib/block_scout_web/templates/transaction/_tabs.html.eex:17 #: lib/block_scout_web/templates/transaction_log/index.html.eex:8 #: lib/block_scout_web/views/address_view.ex:387 -#: lib/block_scout_web/views/transaction_view.ex:534 +#: lib/block_scout_web/views/transaction_view.ex:536 #, elixir-autogen, elixir-format msgid "Logs" msgstr "" @@ -1752,7 +1752,7 @@ msgstr "" msgid "Max Priority Fee per Gas" msgstr "" -#: lib/block_scout_web/views/transaction_view.ex:325 +#: lib/block_scout_web/views/transaction_view.ex:327 #, elixir-autogen, elixir-format msgid "Max of" msgstr "" @@ -2064,8 +2064,8 @@ msgid "Parent Hash" msgstr "" #: lib/block_scout_web/templates/layout/_topnav.html.eex:63 -#: lib/block_scout_web/views/transaction_view.ex:368 -#: lib/block_scout_web/views/transaction_view.ex:407 +#: lib/block_scout_web/views/transaction_view.ex:370 +#: lib/block_scout_web/views/transaction_view.ex:409 #, elixir-autogen, elixir-format msgid "Pending" msgstr "" @@ -2201,7 +2201,7 @@ msgstr "" #: lib/block_scout_web/templates/transaction/_tabs.html.eex:24 #: lib/block_scout_web/templates/transaction_raw_trace/_card_body.html.eex:1 -#: lib/block_scout_web/views/transaction_view.ex:535 +#: lib/block_scout_web/views/transaction_view.ex:537 #, elixir-autogen, elixir-format msgid "Raw Trace" msgstr "" @@ -2514,7 +2514,7 @@ msgstr "" #: lib/block_scout_web/templates/transaction/_tabs.html.eex:29 #: lib/block_scout_web/templates/transaction_state/index.html.eex:6 -#: lib/block_scout_web/views/transaction_view.ex:536 +#: lib/block_scout_web/views/transaction_view.ex:538 #, elixir-autogen, elixir-format msgid "State changes" msgstr "" @@ -2540,7 +2540,7 @@ msgid "Submit an Issue" msgstr "" #: lib/block_scout_web/templates/transaction/_emission_reward_tile.html.eex:8 -#: lib/block_scout_web/views/transaction_view.ex:370 +#: lib/block_scout_web/views/transaction_view.ex:372 #, elixir-autogen, elixir-format msgid "Success" msgstr "" @@ -2849,13 +2849,13 @@ msgid "Token" msgstr "" #: lib/block_scout_web/templates/common_components/_token_transfer_type_display_name.html.eex:3 -#: lib/block_scout_web/views/transaction_view.ex:469 +#: lib/block_scout_web/views/transaction_view.ex:471 #, elixir-autogen, elixir-format msgid "Token Burning" msgstr "" #: lib/block_scout_web/templates/common_components/_token_transfer_type_display_name.html.eex:7 -#: lib/block_scout_web/views/transaction_view.ex:470 +#: lib/block_scout_web/views/transaction_view.ex:472 #, elixir-autogen, elixir-format msgid "Token Creation" msgstr "" @@ -2883,14 +2883,14 @@ msgid "Token ID" msgstr "" #: lib/block_scout_web/templates/common_components/_token_transfer_type_display_name.html.eex:5 -#: lib/block_scout_web/views/transaction_view.ex:468 +#: lib/block_scout_web/views/transaction_view.ex:470 #, elixir-autogen, elixir-format msgid "Token Minting" msgstr "" #: lib/block_scout_web/templates/common_components/_token_transfer_type_display_name.html.eex:9 #: lib/block_scout_web/templates/common_components/_token_transfer_type_display_name.html.eex:11 -#: lib/block_scout_web/views/transaction_view.ex:471 +#: lib/block_scout_web/views/transaction_view.ex:473 #, elixir-autogen, elixir-format msgid "Token Transfer" msgstr "" @@ -2906,7 +2906,7 @@ msgstr "" #: lib/block_scout_web/views/address_view.ex:378 #: lib/block_scout_web/views/tokens/instance/overview_view.ex:114 #: lib/block_scout_web/views/tokens/overview_view.ex:40 -#: lib/block_scout_web/views/transaction_view.ex:532 +#: lib/block_scout_web/views/transaction_view.ex:534 #, elixir-autogen, elixir-format msgid "Token Transfers" msgstr "" @@ -3022,7 +3022,7 @@ msgstr "" #: lib/block_scout_web/templates/account/tag_transaction/form.html.eex:11 #: lib/block_scout_web/templates/account/tag_transaction/index.html.eex:23 #: lib/block_scout_web/templates/address_logs/_logs.html.eex:19 -#: lib/block_scout_web/views/transaction_view.ex:481 +#: lib/block_scout_web/views/transaction_view.ex:483 #, elixir-autogen, elixir-format msgid "Transaction" msgstr "" @@ -3177,7 +3177,7 @@ msgstr "" msgid "Uncles" msgstr "" -#: lib/block_scout_web/views/transaction_view.ex:361 +#: lib/block_scout_web/views/transaction_view.ex:363 #, elixir-autogen, elixir-format msgid "Unconfirmed" msgstr "" diff --git a/apps/block_scout_web/priv/gettext/en/LC_MESSAGES/default.po b/apps/block_scout_web/priv/gettext/en/LC_MESSAGES/default.po index ef9ebe1481..4d1362f4d2 100644 --- a/apps/block_scout_web/priv/gettext/en/LC_MESSAGES/default.po +++ b/apps/block_scout_web/priv/gettext/en/LC_MESSAGES/default.po @@ -68,7 +68,7 @@ msgstr "" msgid "%{subnetwork} Explorer - BlockScout" msgstr "" -#: lib/block_scout_web/views/transaction_view.ex:369 +#: lib/block_scout_web/views/transaction_view.ex:371 #, elixir-autogen, elixir-format msgid "(Awaiting internal transactions for status)" msgstr "" @@ -698,7 +698,7 @@ msgstr "" msgid "Compiler version" msgstr "" -#: lib/block_scout_web/views/transaction_view.ex:362 +#: lib/block_scout_web/views/transaction_view.ex:364 #, elixir-autogen, elixir-format msgid "Confirmed" msgstr "" @@ -783,12 +783,12 @@ msgstr "" msgid "Contract Address Pending" msgstr "" -#: lib/block_scout_web/views/transaction_view.ex:478 +#: lib/block_scout_web/views/transaction_view.ex:480 #, elixir-autogen, elixir-format msgid "Contract Call" msgstr "" -#: lib/block_scout_web/views/transaction_view.ex:475 +#: lib/block_scout_web/views/transaction_view.ex:477 #, elixir-autogen, elixir-format msgid "Contract Creation" msgstr "" @@ -1186,12 +1186,12 @@ msgstr "" msgid "EIP-1167" msgstr "" -#: lib/block_scout_web/views/transaction_view.ex:220 +#: lib/block_scout_web/views/transaction_view.ex:222 #, elixir-autogen, elixir-format msgid "ERC-1155 " msgstr "" -#: lib/block_scout_web/views/transaction_view.ex:218 +#: lib/block_scout_web/views/transaction_view.ex:220 #, elixir-autogen, elixir-format msgid "ERC-20 " msgstr "" @@ -1201,7 +1201,7 @@ msgstr "" msgid "ERC-20 tokens (beta)" msgstr "" -#: lib/block_scout_web/views/transaction_view.ex:219 +#: lib/block_scout_web/views/transaction_view.ex:221 #, elixir-autogen, elixir-format msgid "ERC-721 " msgstr "" @@ -1292,12 +1292,12 @@ msgstr "" msgid "Error trying to fetch balances." msgstr "" -#: lib/block_scout_web/views/transaction_view.ex:373 +#: lib/block_scout_web/views/transaction_view.ex:375 #, elixir-autogen, elixir-format msgid "Error: %{reason}" msgstr "" -#: lib/block_scout_web/views/transaction_view.ex:371 +#: lib/block_scout_web/views/transaction_view.ex:373 #, elixir-autogen, elixir-format msgid "Error: (Awaiting internal transactions for reason)" msgstr "" @@ -1602,7 +1602,7 @@ msgstr "" #: lib/block_scout_web/templates/transaction/_tabs.html.eex:11 #: lib/block_scout_web/templates/transaction_internal_transaction/index.html.eex:6 #: lib/block_scout_web/views/address_view.ex:376 -#: lib/block_scout_web/views/transaction_view.ex:533 +#: lib/block_scout_web/views/transaction_view.ex:535 #, elixir-autogen, elixir-format msgid "Internal Transactions" msgstr "" @@ -1719,7 +1719,7 @@ msgstr "" #: lib/block_scout_web/templates/transaction/_tabs.html.eex:17 #: lib/block_scout_web/templates/transaction_log/index.html.eex:8 #: lib/block_scout_web/views/address_view.ex:387 -#: lib/block_scout_web/views/transaction_view.ex:534 +#: lib/block_scout_web/views/transaction_view.ex:536 #, elixir-autogen, elixir-format msgid "Logs" msgstr "" @@ -1752,7 +1752,7 @@ msgstr "" msgid "Max Priority Fee per Gas" msgstr "" -#: lib/block_scout_web/views/transaction_view.ex:325 +#: lib/block_scout_web/views/transaction_view.ex:327 #, elixir-autogen, elixir-format msgid "Max of" msgstr "" @@ -2064,8 +2064,8 @@ msgid "Parent Hash" msgstr "" #: lib/block_scout_web/templates/layout/_topnav.html.eex:63 -#: lib/block_scout_web/views/transaction_view.ex:368 -#: lib/block_scout_web/views/transaction_view.ex:407 +#: lib/block_scout_web/views/transaction_view.ex:370 +#: lib/block_scout_web/views/transaction_view.ex:409 #, elixir-autogen, elixir-format msgid "Pending" msgstr "" @@ -2201,7 +2201,7 @@ msgstr "" #: lib/block_scout_web/templates/transaction/_tabs.html.eex:24 #: lib/block_scout_web/templates/transaction_raw_trace/_card_body.html.eex:1 -#: lib/block_scout_web/views/transaction_view.ex:535 +#: lib/block_scout_web/views/transaction_view.ex:537 #, elixir-autogen, elixir-format msgid "Raw Trace" msgstr "" @@ -2514,7 +2514,7 @@ msgstr "" #: lib/block_scout_web/templates/transaction/_tabs.html.eex:29 #: lib/block_scout_web/templates/transaction_state/index.html.eex:6 -#: lib/block_scout_web/views/transaction_view.ex:536 +#: lib/block_scout_web/views/transaction_view.ex:538 #, elixir-autogen, elixir-format msgid "State changes" msgstr "" @@ -2540,7 +2540,7 @@ msgid "Submit an Issue" msgstr "" #: lib/block_scout_web/templates/transaction/_emission_reward_tile.html.eex:8 -#: lib/block_scout_web/views/transaction_view.ex:370 +#: lib/block_scout_web/views/transaction_view.ex:372 #, elixir-autogen, elixir-format msgid "Success" msgstr "" @@ -2849,13 +2849,13 @@ msgid "Token" msgstr "" #: lib/block_scout_web/templates/common_components/_token_transfer_type_display_name.html.eex:3 -#: lib/block_scout_web/views/transaction_view.ex:469 +#: lib/block_scout_web/views/transaction_view.ex:471 #, elixir-autogen, elixir-format msgid "Token Burning" msgstr "" #: lib/block_scout_web/templates/common_components/_token_transfer_type_display_name.html.eex:7 -#: lib/block_scout_web/views/transaction_view.ex:470 +#: lib/block_scout_web/views/transaction_view.ex:472 #, elixir-autogen, elixir-format msgid "Token Creation" msgstr "" @@ -2883,14 +2883,14 @@ msgid "Token ID" msgstr "" #: lib/block_scout_web/templates/common_components/_token_transfer_type_display_name.html.eex:5 -#: lib/block_scout_web/views/transaction_view.ex:468 +#: lib/block_scout_web/views/transaction_view.ex:470 #, elixir-autogen, elixir-format msgid "Token Minting" msgstr "" #: lib/block_scout_web/templates/common_components/_token_transfer_type_display_name.html.eex:9 #: lib/block_scout_web/templates/common_components/_token_transfer_type_display_name.html.eex:11 -#: lib/block_scout_web/views/transaction_view.ex:471 +#: lib/block_scout_web/views/transaction_view.ex:473 #, elixir-autogen, elixir-format msgid "Token Transfer" msgstr "" @@ -2906,7 +2906,7 @@ msgstr "" #: lib/block_scout_web/views/address_view.ex:378 #: lib/block_scout_web/views/tokens/instance/overview_view.ex:114 #: lib/block_scout_web/views/tokens/overview_view.ex:40 -#: lib/block_scout_web/views/transaction_view.ex:532 +#: lib/block_scout_web/views/transaction_view.ex:534 #, elixir-autogen, elixir-format msgid "Token Transfers" msgstr "" @@ -3022,7 +3022,7 @@ msgstr "" #: lib/block_scout_web/templates/account/tag_transaction/form.html.eex:11 #: lib/block_scout_web/templates/account/tag_transaction/index.html.eex:23 #: lib/block_scout_web/templates/address_logs/_logs.html.eex:19 -#: lib/block_scout_web/views/transaction_view.ex:481 +#: lib/block_scout_web/views/transaction_view.ex:483 #, elixir-autogen, elixir-format msgid "Transaction" msgstr "" @@ -3177,7 +3177,7 @@ msgstr "" msgid "Uncles" msgstr "" -#: lib/block_scout_web/views/transaction_view.ex:361 +#: lib/block_scout_web/views/transaction_view.ex:363 #, elixir-autogen, elixir-format msgid "Unconfirmed" msgstr "" diff --git a/apps/block_scout_web/test/block_scout_web/channels/websocket_v2_test.exs b/apps/block_scout_web/test/block_scout_web/channels/websocket_v2_test.exs index 918b636cb3..bfb0424424 100644 --- a/apps/block_scout_web/test/block_scout_web/channels/websocket_v2_test.exs +++ b/apps/block_scout_web/test/block_scout_web/channels/websocket_v2_test.exs @@ -74,6 +74,7 @@ defmodule BlockScoutWeb.WebsocketV2Test do %{ block_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd", block_number: 37, + block_timestamp: Timex.parse!("2017-12-15T21:06:30.000000Z", "{ISO:Extended:Z}"), cumulative_gas_used: 50450, from_address_hash: "0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca", gas: 4_700_000, @@ -96,6 +97,7 @@ defmodule BlockScoutWeb.WebsocketV2Test do %{ block_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd", block_number: 37, + block_timestamp: Timex.parse!("2017-12-15T21:06:30.000000Z", "{ISO:Extended:Z}"), cumulative_gas_used: 50450, from_address_hash: "0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca", gas: 4_700_000, diff --git a/apps/block_scout_web/test/block_scout_web/controllers/api/v2/search_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/api/v2/search_controller_test.exs index ee9ee43a03..95584c013b 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/api/v2/search_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/api/v2/search_controller_test.exs @@ -202,7 +202,7 @@ defmodule BlockScoutWeb.API.V2.SearchControllerTest do end test "search transaction", %{conn: conn} do - tx = insert(:transaction) + tx = insert(:transaction, block_timestamp: nil) request = get(conn, "/api/v2/search?q=#{tx.hash}") assert response = json_response(request, 200) diff --git a/apps/explorer/config/config.exs b/apps/explorer/config/config.exs index 500417a2fd..88d38c68a9 100644 --- a/apps/explorer/config/config.exs +++ b/apps/explorer/config/config.exs @@ -113,6 +113,8 @@ config :explorer, Explorer.TokenTransferTokenIdMigration.Supervisor, enabled: tr config :explorer, Explorer.TokenInstanceOwnerAddressMigration.Supervisor, enabled: true +config :explorer, Explorer.TransactionsDenormalizationMigrator, enabled: true + config :explorer, Explorer.Chain.Fetcher.CheckBytecodeMatchingOnDemand, enabled: true config :explorer, Explorer.Chain.Fetcher.FetchValidatorInfoOnDemand, enabled: true diff --git a/apps/explorer/config/runtime/test.exs b/apps/explorer/config/runtime/test.exs index 9449463a5b..37ce93b2cb 100644 --- a/apps/explorer/config/runtime/test.exs +++ b/apps/explorer/config/runtime/test.exs @@ -37,6 +37,8 @@ config :explorer, Explorer.TokenTransferTokenIdMigration.Supervisor, enabled: fa config :explorer, Explorer.TokenInstanceOwnerAddressMigration.Supervisor, enabled: false +config :explorer, Explorer.TransactionsDenormalizationMigrator, enabled: false + config :explorer, realtime_events_sender: Explorer.Chain.Events.SimpleSender diff --git a/apps/explorer/lib/explorer/application.ex b/apps/explorer/lib/explorer/application.ex index 666407fbc0..497043d143 100644 --- a/apps/explorer/lib/explorer/application.ex +++ b/apps/explorer/lib/explorer/application.ex @@ -12,6 +12,7 @@ defmodule Explorer.Application do AddressesTabsCounters, AddressSum, AddressSumMinusBurnt, + BackgroundMigrations, Block, BlockNumber, Blocks, @@ -63,6 +64,7 @@ defmodule Explorer.Application do Accounts, AddressSum, AddressSumMinusBurnt, + BackgroundMigrations, Block, BlockNumber, Blocks, @@ -125,7 +127,8 @@ defmodule Explorer.Application do configure(Explorer.Chain.Fetcher.FetchValidatorInfoOnDemand), configure(Explorer.TokenInstanceOwnerAddressMigration.Supervisor), sc_microservice_configure(Explorer.Chain.Fetcher.LookUpSmartContractSourcesOnDemand), - configure(Explorer.Chain.Cache.RootstockLockedBTC) + configure(Explorer.Chain.Cache.RootstockLockedBTC), + configure(Explorer.TransactionsDenormalizationMigrator) ] |> List.flatten() diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index af2d6874a6..b6d27141d7 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -51,6 +51,7 @@ defmodule Explorer.Chain do CurrencyHelper, Data, DecompiledSmartContract, + DenormalizationHelper, Hash, Import, InternalTransaction, @@ -521,12 +522,22 @@ defmodule Explorer.Chain do @spec gas_payment_by_block_hash([Hash.Full.t()]) :: %{Hash.Full.t() => Wei.t()} def gas_payment_by_block_hash(block_hashes) when is_list(block_hashes) do query = - from( - transaction in Transaction, - where: transaction.block_hash in ^block_hashes and transaction.block_consensus == true, - group_by: transaction.block_hash, - select: {transaction.block_hash, %Wei{value: coalesce(sum(transaction.gas_used * transaction.gas_price), 0)}} - ) + if DenormalizationHelper.denormalization_finished?() do + from( + transaction in Transaction, + where: transaction.block_hash in ^block_hashes and transaction.block_consensus == true, + group_by: transaction.block_hash, + select: {transaction.block_hash, %Wei{value: coalesce(sum(transaction.gas_used * transaction.gas_price), 0)}} + ) + else + from( + block in Block, + left_join: transaction in assoc(block, :transactions), + where: block.hash in ^block_hashes and block.consensus == true, + group_by: block.hash, + select: {block.hash, %Wei{value: coalesce(sum(transaction.gas_used * transaction.gas_price), 0)}} + ) + end query |> Repo.all() diff --git a/apps/explorer/lib/explorer/chain/cache/background_migrations.ex b/apps/explorer/lib/explorer/chain/cache/background_migrations.ex new file mode 100644 index 0000000000..4b33076a96 --- /dev/null +++ b/apps/explorer/lib/explorer/chain/cache/background_migrations.ex @@ -0,0 +1,23 @@ +defmodule Explorer.Chain.Cache.BackgroundMigrations do + @moduledoc """ + Caches background migrations' status. + """ + + require Logger + + use Explorer.Chain.MapCache, + name: :background_migrations_status, + key: :denormalization_finished + + @dialyzer :no_match + + alias Explorer.TransactionsDenormalizationMigrator + + defp handle_fallback(:denormalization_finished) do + Task.start(fn -> + set_denormalization_finished(TransactionsDenormalizationMigrator.migration_finished?()) + end) + + {:return, false} + end +end diff --git a/apps/explorer/lib/explorer/chain/csv_export/address_token_transfer_csv_exporter.ex b/apps/explorer/lib/explorer/chain/csv_export/address_token_transfer_csv_exporter.ex index 7042297572..4409d1475e 100644 --- a/apps/explorer/lib/explorer/chain/csv_export/address_token_transfer_csv_exporter.ex +++ b/apps/explorer/lib/explorer/chain/csv_export/address_token_transfer_csv_exporter.ex @@ -12,7 +12,7 @@ defmodule Explorer.Chain.CSVExport.AddressTokenTransferCsvExporter do ] alias Explorer.{Chain, PagingOptions, Repo} - alias Explorer.Chain.{Address, Hash, TokenTransfer} + alias Explorer.Chain.{Address, Hash, TokenTransfer, Transaction} alias Explorer.Chain.CSVExport.Helper @paging_options %PagingOptions{page_size: Helper.limit(), asc_order: true} @@ -68,7 +68,7 @@ defmodule Explorer.Chain.CSVExport.AddressTokenTransferCsvExporter do [ to_string(token_transfer.transaction_hash), token_transfer.transaction.block_number, - token_transfer.transaction.block.timestamp, + Transaction.block_timestamp(token_transfer.transaction), Address.checksum(token_transfer.from_address_hash), Address.checksum(token_transfer.to_address_hash), Address.checksum(token_transfer.token_contract_address_hash), diff --git a/apps/explorer/lib/explorer/chain/csv_export/address_transaction_csv_exporter.ex b/apps/explorer/lib/explorer/chain/csv_export/address_transaction_csv_exporter.ex index a0404f4773..600d53d63a 100644 --- a/apps/explorer/lib/explorer/chain/csv_export/address_transaction_csv_exporter.ex +++ b/apps/explorer/lib/explorer/chain/csv_export/address_transaction_csv_exporter.ex @@ -10,15 +10,9 @@ defmodule Explorer.Chain.CSVExport.AddressTransactionCsvExporter do alias Explorer.{Chain, Market, PagingOptions, Repo} alias Explorer.Market.MarketHistory - alias Explorer.Chain.{Address, Hash, Transaction, Wei} + alias Explorer.Chain.{Address, DenormalizationHelper, Hash, Transaction, Wei} alias Explorer.Chain.CSVExport.Helper - @necessity_by_association [ - necessity_by_association: %{ - :block => :required - } - ] - @paging_options %PagingOptions{page_size: Helper.limit()} @spec export(Hash.Address.t(), String.t(), String.t(), String.t() | nil, String.t() | nil) :: Enumerable.t() @@ -35,7 +29,8 @@ defmodule Explorer.Chain.CSVExport.AddressTransactionCsvExporter do # sobelow_skip ["DOS.StringToAtom"] def fetch_transactions(address_hash, from_block, to_block, filter_type, filter_value, paging_options) do options = - @necessity_by_association + [] + |> DenormalizationHelper.extend_block_necessity(:required) |> Keyword.put(:paging_options, paging_options) |> Keyword.put(:from_block, from_block) |> Keyword.put(:to_block, to_block) @@ -67,7 +62,7 @@ defmodule Explorer.Chain.CSVExport.AddressTransactionCsvExporter do date_to_prices = Enum.reduce(transactions, %{}, fn tx, acc -> - date = DateTime.to_date(tx.block.timestamp) + date = tx |> Transaction.block_timestamp() |> DateTime.to_date() if Map.has_key?(acc, date) do acc @@ -79,12 +74,12 @@ defmodule Explorer.Chain.CSVExport.AddressTransactionCsvExporter do transaction_lists = transactions |> Stream.map(fn transaction -> - {opening_price, closing_price} = date_to_prices[DateTime.to_date(transaction.block.timestamp)] + {opening_price, closing_price} = date_to_prices[DateTime.to_date(Transaction.block_timestamp(transaction))] [ to_string(transaction.hash), transaction.block_number, - transaction.block.timestamp, + Transaction.block_timestamp(transaction), Address.checksum(transaction.from_address_hash), Address.checksum(transaction.to_address_hash), Address.checksum(transaction.created_contract_address_hash), diff --git a/apps/explorer/lib/explorer/chain/denormalization_helper.ex b/apps/explorer/lib/explorer/chain/denormalization_helper.ex new file mode 100644 index 0000000000..284ccf64a2 --- /dev/null +++ b/apps/explorer/lib/explorer/chain/denormalization_helper.ex @@ -0,0 +1,44 @@ +defmodule Explorer.Chain.DenormalizationHelper do + @moduledoc false + + alias Explorer.Chain.Cache.BackgroundMigrations + + def extend_block_necessity(opts, necessity \\ :optional) do + if denormalization_finished?() do + opts + else + Keyword.update(opts, :necessity_by_association, %{:block => necessity}, &Map.put(&1, :block, necessity)) + end + end + + def extend_transaction_block_necessity(opts, necessity \\ :optional) do + if denormalization_finished?() do + opts + else + Keyword.update( + opts, + :necessity_by_association, + %{[transaction: :block] => necessity}, + &(&1 |> Map.delete(:transaction) |> Map.put([transaction: :block], necessity)) + ) + end + end + + def extend_transaction_preload(preloads) do + if denormalization_finished?() do + preloads + else + [transaction: :block] ++ (preloads -- [:transaction]) + end + end + + def extend_block_preload(preloads) do + if denormalization_finished?() do + preloads + else + [:block | preloads] + end + end + + def denormalization_finished?, do: BackgroundMigrations.get_denormalization_finished() +end diff --git a/apps/explorer/lib/explorer/chain/import/runner/transactions.ex b/apps/explorer/lib/explorer/chain/import/runner/transactions.ex index b9927a5c5e..62578969a7 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/transactions.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/transactions.ex @@ -161,7 +161,7 @@ defmodule Explorer.Chain.Import.Runner.Transactions do ], where: fragment( - "(EXCLUDED.block_hash, EXCLUDED.block_number, EXCLUDED.block_consensus, EXCLUDED.block_timestamp, EXCLUDED.created_contract_address_hash, EXCLUDED.created_contract_code_indexed_at, EXCLUDED.cumulative_gas_used, EXCLUDED.from_address_hash, EXCLUDED.gas, EXCLUDED.gas_price, EXCLUDED.gas_used, EXCLUDED.index, EXCLUDED.input, EXCLUDED.nonce, EXCLUDED.r, EXCLUDED.s, EXCLUDED.status, EXCLUDED.to_address_hash, EXCLUDED.v, EXCLUDED.value, EXCLUDED.earliest_processing_start, EXCLUDED.revert_reason, EXCLUDED.max_priority_fee_per_gas, EXCLUDED.max_fee_per_gas, EXCLUDED.type, EXCLUDED.execution_node_hash, EXCLUDED.wrapped_type, EXCLUDED.wrapped_nonce, EXCLUDED.wrapped_to_address_hash, EXCLUDED.wrapped_gas, EXCLUDED.wrapped_gas_price, EXCLUDED.wrapped_max_priority_fee_per_gas, EXCLUDED.wrapped_max_fee_per_gas, EXCLUDED.wrapped_value, EXCLUDED.wrapped_input, EXCLUDED.wrapped_v, EXCLUDED.wrapped_r, EXCLUDED.wrapped_s, EXCLUDED.wrapped_hash) IS DISTINCT FROM (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + "(EXCLUDED.block_hash, EXCLUDED.block_number, EXCLUDED.block_consensus, EXCLUDED.block_timestamp, EXCLUDED.created_contract_address_hash, EXCLUDED.created_contract_code_indexed_at, EXCLUDED.cumulative_gas_used, EXCLUDED.from_address_hash, EXCLUDED.gas, EXCLUDED.gas_price, EXCLUDED.gas_used, EXCLUDED.index, EXCLUDED.input, EXCLUDED.nonce, EXCLUDED.r, EXCLUDED.s, EXCLUDED.status, EXCLUDED.to_address_hash, EXCLUDED.v, EXCLUDED.value, EXCLUDED.earliest_processing_start, EXCLUDED.revert_reason, EXCLUDED.max_priority_fee_per_gas, EXCLUDED.max_fee_per_gas, EXCLUDED.type, EXCLUDED.execution_node_hash, EXCLUDED.wrapped_type, EXCLUDED.wrapped_nonce, EXCLUDED.wrapped_to_address_hash, EXCLUDED.wrapped_gas, EXCLUDED.wrapped_gas_price, EXCLUDED.wrapped_max_priority_fee_per_gas, EXCLUDED.wrapped_max_fee_per_gas, EXCLUDED.wrapped_value, EXCLUDED.wrapped_input, EXCLUDED.wrapped_v, EXCLUDED.wrapped_r, EXCLUDED.wrapped_s, EXCLUDED.wrapped_hash) IS DISTINCT FROM (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", transaction.block_hash, transaction.block_number, transaction.block_consensus, @@ -242,7 +242,7 @@ defmodule Explorer.Chain.Import.Runner.Transactions do ], where: fragment( - "(EXCLUDED.block_hash, EXCLUDED.block_number, EXCLUDED.block_consensus, EXCLUDED.block_timestamp, EXCLUDED.created_contract_address_hash, EXCLUDED.created_contract_code_indexed_at, EXCLUDED.cumulative_gas_used, EXCLUDED.from_address_hash, EXCLUDED.gas, EXCLUDED.gas_price, EXCLUDED.gas_used, EXCLUDED.index, EXCLUDED.input, EXCLUDED.nonce, EXCLUDED.r, EXCLUDED.s, EXCLUDED.status, EXCLUDED.to_address_hash, EXCLUDED.v, EXCLUDED.value, EXCLUDED.earliest_processing_start, EXCLUDED.revert_reason, EXCLUDED.max_priority_fee_per_gas, EXCLUDED.max_fee_per_gas, EXCLUDED.type) IS DISTINCT FROM (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + "(EXCLUDED.block_hash, EXCLUDED.block_number, EXCLUDED.block_consensus, EXCLUDED.block_timestamp, EXCLUDED.created_contract_address_hash, EXCLUDED.created_contract_code_indexed_at, EXCLUDED.cumulative_gas_used, EXCLUDED.from_address_hash, EXCLUDED.gas, EXCLUDED.gas_price, EXCLUDED.gas_used, EXCLUDED.index, EXCLUDED.input, EXCLUDED.nonce, EXCLUDED.r, EXCLUDED.s, EXCLUDED.status, EXCLUDED.to_address_hash, EXCLUDED.v, EXCLUDED.value, EXCLUDED.earliest_processing_start, EXCLUDED.revert_reason, EXCLUDED.max_priority_fee_per_gas, EXCLUDED.max_fee_per_gas, EXCLUDED.type) IS DISTINCT FROM (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", transaction.block_hash, transaction.block_number, transaction.block_consensus, diff --git a/apps/explorer/lib/explorer/chain/search.ex b/apps/explorer/lib/explorer/chain/search.ex index d74f697a9b..20082bccbf 100644 --- a/apps/explorer/lib/explorer/chain/search.ex +++ b/apps/explorer/lib/explorer/chain/search.ex @@ -21,6 +21,7 @@ defmodule Explorer.Chain.Search do alias Explorer.Chain.{ Address, Block, + DenormalizationHelper, SmartContract, Token, Transaction @@ -382,38 +383,72 @@ defmodule Explorer.Chain.Search do end defp search_tx_query(term) do - case Chain.string_to_transaction_hash(term) do - {:ok, tx_hash} -> - transaction_search_fields = %{ - address_hash: dynamic([_, _], type(^nil, :binary)), - tx_hash: dynamic([transaction, _], transaction.hash), - block_hash: dynamic([_, _], type(^nil, :binary)), - type: "transaction", - name: nil, - symbol: nil, - holder_count: nil, - inserted_at: dynamic([transaction, _], transaction.inserted_at), - block_number: 0, - icon_url: nil, - token_type: nil, - timestamp: dynamic([_, block], block.timestamp), - verified: nil, - exchange_rate: nil, - total_supply: nil, - circulating_market_cap: nil, - priority: 0, - is_verified_via_admin_panel: nil - } + if DenormalizationHelper.denormalization_finished?() do + case Chain.string_to_transaction_hash(term) do + {:ok, tx_hash} -> + transaction_search_fields = %{ + address_hash: dynamic([_], type(^nil, :binary)), + tx_hash: dynamic([transaction], transaction.hash), + block_hash: dynamic([_], type(^nil, :binary)), + type: "transaction", + name: nil, + symbol: nil, + holder_count: nil, + inserted_at: dynamic([transaction], transaction.inserted_at), + block_number: 0, + icon_url: nil, + token_type: nil, + timestamp: dynamic([transaction], transaction.block_timestamp), + verified: nil, + exchange_rate: nil, + total_supply: nil, + circulating_market_cap: nil, + priority: 0, + is_verified_via_admin_panel: nil + } + + from(transaction in Transaction, + where: transaction.hash == ^tx_hash, + select: ^transaction_search_fields + ) - from(transaction in Transaction, - left_join: block in Block, - on: transaction.block_hash == block.hash, - where: transaction.hash == ^tx_hash, - select: ^transaction_search_fields - ) + _ -> + nil + end + else + case Chain.string_to_transaction_hash(term) do + {:ok, tx_hash} -> + transaction_search_fields = %{ + address_hash: dynamic([_, _], type(^nil, :binary)), + tx_hash: dynamic([transaction, _], transaction.hash), + block_hash: dynamic([_, _], type(^nil, :binary)), + type: "transaction", + name: nil, + symbol: nil, + holder_count: nil, + inserted_at: dynamic([transaction, _], transaction.inserted_at), + block_number: 0, + icon_url: nil, + token_type: nil, + timestamp: dynamic([_, block], block.timestamp), + verified: nil, + exchange_rate: nil, + total_supply: nil, + circulating_market_cap: nil, + priority: 0, + is_verified_via_admin_panel: nil + } + + from(transaction in Transaction, + left_join: block in Block, + on: transaction.block_hash == block.hash, + where: transaction.hash == ^tx_hash, + select: ^transaction_search_fields + ) - _ -> - nil + _ -> + nil + end end end diff --git a/apps/explorer/lib/explorer/chain/token_transfer.ex b/apps/explorer/lib/explorer/chain/token_transfer.ex index b1663a1e32..8b9b878914 100644 --- a/apps/explorer/lib/explorer/chain/token_transfer.ex +++ b/apps/explorer/lib/explorer/chain/token_transfer.ex @@ -28,7 +28,7 @@ defmodule Explorer.Chain.TokenTransfer do import Ecto.Query, only: [from: 2, limit: 2, where: 3, join: 5, order_by: 3, preload: 3] alias Explorer.Chain - alias Explorer.Chain.{Address, Block, Hash, TokenTransfer, Transaction} + alias Explorer.Chain.{Address, Block, DenormalizationHelper, Hash, TokenTransfer, Transaction} alias Explorer.Chain.Token.Instance alias Explorer.{PagingOptions, Repo} @@ -161,12 +161,13 @@ defmodule Explorer.Chain.TokenTransfer do @spec fetch_token_transfers_from_token_hash(Hash.t(), [paging_options | api?]) :: [] def fetch_token_transfers_from_token_hash(token_address_hash, options) do paging_options = Keyword.get(options, :paging_options, @default_paging_options) + preloads = DenormalizationHelper.extend_transaction_preload([:transaction, :token, :from_address, :to_address]) query = from( tt in TokenTransfer, where: tt.token_contract_address_hash == ^token_address_hash and not is_nil(tt.block_number), - preload: [:transaction, :token, :from_address, :to_address], + preload: ^preloads, order_by: [desc: tt.block_number, desc: tt.log_index] ) @@ -179,6 +180,7 @@ defmodule Explorer.Chain.TokenTransfer do @spec fetch_token_transfers_from_token_hash_and_token_id(Hash.t(), non_neg_integer(), [paging_options | api?]) :: [] def fetch_token_transfers_from_token_hash_and_token_id(token_address_hash, token_id, options) do paging_options = Keyword.get(options, :paging_options, @default_paging_options) + preloads = DenormalizationHelper.extend_transaction_preload([:transaction, :token, :from_address, :to_address]) query = from( @@ -186,7 +188,7 @@ defmodule Explorer.Chain.TokenTransfer do where: tt.token_contract_address_hash == ^token_address_hash, where: fragment("? @> ARRAY[?::decimal]", tt.token_ids, ^Decimal.new(token_id)), where: not is_nil(tt.block_number), - preload: [:transaction, :token, :from_address, :to_address], + preload: ^preloads, order_by: [desc: tt.block_number, desc: tt.log_index] ) diff --git a/apps/explorer/lib/explorer/chain/transaction.ex b/apps/explorer/lib/explorer/chain/transaction.ex index f8804f06fa..a77700db8e 100644 --- a/apps/explorer/lib/explorer/chain/transaction.ex +++ b/apps/explorer/lib/explorer/chain/transaction.ex @@ -17,6 +17,7 @@ defmodule Explorer.Chain.Transaction do Block, ContractMethod, Data, + DenormalizationHelper, Gas, Hash, InternalTransaction, @@ -552,6 +553,11 @@ defmodule Explorer.Chain.Transaction do |> unique_constraint(:hash) end + def block_timestamp(%{block_number: nil, inserted_at: time}), do: time + def block_timestamp(%{block_timestamp: time}) when not is_nil(time), do: time + def block_timestamp(%{block: %{timestamp: time}}), do: time + def block_timestamp(_), do: nil + def preload_token_transfers(query, address_hash) do token_transfers_query = from( @@ -1027,11 +1033,12 @@ defmodule Explorer.Chain.Transaction do """ def transactions_with_token_transfers(address_hash, token_hash) do query = transactions_with_token_transfers_query(address_hash, token_hash) + preloads = DenormalizationHelper.extend_block_preload([:from_address, :to_address, :created_contract_address]) from( t in subquery(query), order_by: [desc: t.block_number, desc: t.index], - preload: [:from_address, :to_address, :created_contract_address] + preload: ^preloads ) end @@ -1048,11 +1055,12 @@ defmodule Explorer.Chain.Transaction do def transactions_with_token_transfers_direction(direction, address_hash) do query = transactions_with_token_transfers_query_direction(direction, address_hash) + preloads = DenormalizationHelper.extend_block_preload([:from_address, :to_address, :created_contract_address]) from( t in subquery(query), order_by: [desc: t.block_number, desc: t.index], - preload: [:from_address, :to_address, :created_contract_address] + preload: ^preloads ) end diff --git a/apps/explorer/lib/explorer/chain/transaction/history/historian.ex b/apps/explorer/lib/explorer/chain/transaction/history/historian.ex index 126b59b299..2c8e4479cf 100644 --- a/apps/explorer/lib/explorer/chain/transaction/history/historian.ex +++ b/apps/explorer/lib/explorer/chain/transaction/history/historian.ex @@ -6,7 +6,7 @@ defmodule Explorer.Chain.Transaction.History.Historian do use Explorer.History.Historian alias Explorer.{Chain, Repo} - alias Explorer.Chain.{Block, Transaction} + alias Explorer.Chain.{Block, DenormalizationHelper, Transaction} alias Explorer.Chain.Events.Publisher alias Explorer.Chain.Transaction.History.TransactionStats alias Explorer.History.Process, as: HistoryProcess @@ -89,22 +89,57 @@ defmodule Explorer.Chain.Transaction.History.Historian do Logger.info("tx/per day chart: min/max block numbers [#{min_block}, #{max_block}]") all_transactions_query = + if DenormalizationHelper.denormalization_finished?() do + from( + transaction in Transaction, + where: transaction.block_number >= ^min_block and transaction.block_number <= ^max_block, + where: transaction.block_consensus == true, + select: transaction + ) + else + from( + transaction in Transaction, + where: transaction.block_number >= ^min_block and transaction.block_number <= ^max_block + ) + end + + all_blocks_query = from( - transaction in Transaction, - where: transaction.block_number >= ^min_block and transaction.block_number <= ^max_block, - where: transaction.block_consensus == true, - select: transaction + block in Block, + where: block.consensus == true, + where: block.number >= ^min_block and block.number <= ^max_block, + select: block.number ) - num_transactions = Repo.aggregate(all_transactions_query, :count, :hash, timeout: :infinity) + query = + if DenormalizationHelper.denormalization_finished?() do + all_transactions_query + else + from(transaction in subquery(all_transactions_query), + join: block in subquery(all_blocks_query), + on: transaction.block_number == block.number, + select: transaction + ) + end + + num_transactions = Repo.aggregate(query, :count, :hash, timeout: :infinity) Logger.info("tx/per day chart: num of transactions #{num_transactions}") - gas_used = Repo.aggregate(all_transactions_query, :sum, :gas_used, timeout: :infinity) + gas_used = Repo.aggregate(query, :sum, :gas_used, timeout: :infinity) Logger.info("tx/per day chart: total gas used #{gas_used}") total_fee_query = - from(transaction in subquery(all_transactions_query), - select: fragment("SUM(? * ?)", transaction.gas_price, transaction.gas_used) - ) + if DenormalizationHelper.denormalization_finished?() do + from(transaction in subquery(all_transactions_query), + select: fragment("SUM(? * ?)", transaction.gas_price, transaction.gas_used) + ) + else + from(transaction in subquery(all_transactions_query), + join: block in Block, + on: transaction.block_hash == block.hash, + where: block.consensus == true, + select: fragment("SUM(? * ?)", transaction.gas_price, transaction.gas_used) + ) + end total_fee = Repo.one(total_fee_query, timeout: :infinity) Logger.info("tx/per day chart: total fee #{total_fee}") diff --git a/apps/explorer/lib/explorer/etherscan.ex b/apps/explorer/lib/explorer/etherscan.ex index f62f206561..dba8be73eb 100644 --- a/apps/explorer/lib/explorer/etherscan.ex +++ b/apps/explorer/lib/explorer/etherscan.ex @@ -9,7 +9,7 @@ defmodule Explorer.Etherscan do alias Explorer.Etherscan.Logs alias Explorer.{Chain, Repo} alias Explorer.Chain.Address.{CurrentTokenBalance, TokenBalance} - alias Explorer.Chain.{Address, Block, Hash, InternalTransaction, TokenTransfer, Transaction} + alias Explorer.Chain.{Address, Block, DenormalizationHelper, Hash, InternalTransaction, TokenTransfer, Transaction} alias Explorer.Chain.Transaction.History.TransactionStats @default_options %{ @@ -101,17 +101,32 @@ defmodule Explorer.Etherscan do @spec list_internal_transactions(Hash.Full.t()) :: [map()] def list_internal_transactions(%Hash{byte_count: unquote(Hash.Full.byte_count())} = transaction_hash) do query = - from( - it in InternalTransaction, - inner_join: transaction in assoc(it, :transaction), - where: it.transaction_hash == ^transaction_hash, - limit: 10_000, - select: - merge(map(it, ^@internal_transaction_fields), %{ - block_timestamp: transaction.block_timestamp, - block_number: transaction.block_number - }) - ) + if DenormalizationHelper.denormalization_finished?() do + from( + it in InternalTransaction, + inner_join: transaction in assoc(it, :transaction), + where: it.transaction_hash == ^transaction_hash, + limit: 10_000, + select: + merge(map(it, ^@internal_transaction_fields), %{ + block_timestamp: transaction.block_timestamp, + block_number: transaction.block_number + }) + ) + else + from( + it in InternalTransaction, + inner_join: t in assoc(it, :transaction), + inner_join: b in assoc(t, :block), + where: it.transaction_hash == ^transaction_hash, + limit: 10_000, + select: + merge(map(it, ^@internal_transaction_fields), %{ + block_timestamp: b.timestamp, + block_number: b.number + }) + ) + end query |> Chain.where_transaction_has_multiple_internal_transactions() @@ -211,18 +226,34 @@ defmodule Explorer.Etherscan do |> Repo.replica().all() else query = - from( - it in InternalTransaction, - inner_join: transaction in assoc(it, :transaction), - order_by: [{^options.order_by_direction, transaction.block_number}], - limit: ^options.page_size, - offset: ^offset(options), - select: - merge(map(it, ^@internal_transaction_fields), %{ - block_timestamp: transaction.block_timestamp, - block_number: transaction.block_number - }) - ) + if DenormalizationHelper.denormalization_finished?() do + from( + it in InternalTransaction, + inner_join: transaction in assoc(it, :transaction), + order_by: [{^options.order_by_direction, transaction.block_number}], + limit: ^options.page_size, + offset: ^offset(options), + select: + merge(map(it, ^@internal_transaction_fields), %{ + block_timestamp: transaction.block_timestamp, + block_number: transaction.block_number + }) + ) + else + from( + it in InternalTransaction, + inner_join: t in assoc(it, :transaction), + inner_join: b in assoc(t, :block), + order_by: [{^options.order_by_direction, t.block_number}], + limit: ^options.page_size, + offset: ^offset(options), + select: + merge(map(it, ^@internal_transaction_fields), %{ + block_timestamp: b.timestamp, + block_number: b.number + }) + ) + end query |> Chain.where_transaction_has_multiple_internal_transactions() @@ -393,16 +424,31 @@ defmodule Explorer.Etherscan do defp list_transactions(address_hash, max_block_number, options) do query = - from( - t in Transaction, - order_by: [{^options.order_by_direction, t.block_number}], - limit: ^options.page_size, - offset: ^offset(options), - select: - merge(map(t, ^@transaction_fields), %{ - confirmations: fragment("? - ?", ^max_block_number, t.block_number) - }) - ) + if DenormalizationHelper.denormalization_finished?() do + from( + t in Transaction, + order_by: [{^options.order_by_direction, t.block_number}], + limit: ^options.page_size, + offset: ^offset(options), + select: + merge(map(t, ^@transaction_fields), %{ + confirmations: fragment("? - ?", ^max_block_number, t.block_number) + }) + ) + else + from( + t in Transaction, + inner_join: b in assoc(t, :block), + order_by: [{^options.order_by_direction, t.block_number}], + limit: ^options.page_size, + offset: ^offset(options), + select: + merge(map(t, ^@transaction_fields), %{ + block_timestamp: b.timestamp, + confirmations: fragment("? - ?", ^max_block_number, t.block_number) + }) + ) + end query |> where_address_match(address_hash, options) @@ -465,37 +511,71 @@ defmodule Explorer.Etherscan do |> where_contract_address_match(contract_address_hash) wrapped_query = - from( - tt in subquery(tt_specific_token_query), - inner_join: t in Transaction, - on: tt.transaction_hash == t.hash and tt.block_number == t.block_number and tt.block_hash == t.block_hash, - order_by: [{^options.order_by_direction, tt.block_number}, {^options.order_by_direction, tt.token_log_index}], - select: %{ - token_contract_address_hash: tt.token_contract_address_hash, - transaction_hash: tt.transaction_hash, - from_address_hash: tt.from_address_hash, - to_address_hash: tt.to_address_hash, - amount: tt.amount, - amounts: tt.amounts, - transaction_nonce: t.nonce, - transaction_index: t.index, - transaction_gas: t.gas, - transaction_gas_price: t.gas_price, - transaction_gas_used: t.gas_used, - transaction_cumulative_gas_used: t.cumulative_gas_used, - transaction_input: t.input, - block_hash: t.block_hash, - block_number: t.block_number, - block_timestamp: t.block_timestamp, - confirmations: fragment("? - ?", ^block_height, t.block_number), - token_ids: tt.token_ids, - token_name: tt.token_name, - token_symbol: tt.token_symbol, - token_decimals: tt.token_decimals, - token_type: tt.token_type, - token_log_index: tt.token_log_index - } - ) + if DenormalizationHelper.denormalization_finished?() do + from( + tt in subquery(tt_specific_token_query), + inner_join: t in Transaction, + on: tt.transaction_hash == t.hash and tt.block_number == t.block_number and tt.block_hash == t.block_hash, + order_by: [{^options.order_by_direction, tt.block_number}, {^options.order_by_direction, tt.token_log_index}], + select: %{ + token_contract_address_hash: tt.token_contract_address_hash, + transaction_hash: tt.transaction_hash, + from_address_hash: tt.from_address_hash, + to_address_hash: tt.to_address_hash, + amount: tt.amount, + amounts: tt.amounts, + transaction_nonce: t.nonce, + transaction_index: t.index, + transaction_gas: t.gas, + transaction_gas_price: t.gas_price, + transaction_gas_used: t.gas_used, + transaction_cumulative_gas_used: t.cumulative_gas_used, + transaction_input: t.input, + block_hash: t.block_hash, + block_number: t.block_number, + block_timestamp: t.block_timestamp, + confirmations: fragment("? - ?", ^block_height, t.block_number), + token_ids: tt.token_ids, + token_name: tt.token_name, + token_symbol: tt.token_symbol, + token_decimals: tt.token_decimals, + token_type: tt.token_type, + token_log_index: tt.token_log_index + } + ) + else + from( + tt in subquery(tt_specific_token_query), + inner_join: t in Transaction, + on: tt.transaction_hash == t.hash and tt.block_number == t.block_number and tt.block_hash == t.block_hash, + inner_join: b in assoc(t, :block), + order_by: [{^options.order_by_direction, tt.block_number}, {^options.order_by_direction, tt.token_log_index}], + select: %{ + token_contract_address_hash: tt.token_contract_address_hash, + transaction_hash: tt.transaction_hash, + from_address_hash: tt.from_address_hash, + to_address_hash: tt.to_address_hash, + amount: tt.amount, + transaction_nonce: t.nonce, + transaction_index: t.index, + transaction_gas: t.gas, + transaction_gas_price: t.gas_price, + transaction_gas_used: t.gas_used, + transaction_cumulative_gas_used: t.cumulative_gas_used, + transaction_input: t.input, + block_hash: b.hash, + block_number: b.number, + block_timestamp: b.timestamp, + confirmations: fragment("? - ?", ^block_height, t.block_number), + token_id: tt.token_id, + token_name: tt.token_name, + token_symbol: tt.token_symbol, + token_decimals: tt.token_decimals, + token_type: tt.token_type, + token_log_index: tt.token_log_index + } + ) + end wrapped_query |> where_start_transaction_block_match(options) @@ -517,26 +597,42 @@ defmodule Explorer.Etherscan do defp where_start_transaction_block_match(query, %{start_block: nil}), do: query - defp where_start_transaction_block_match(query, %{start_block: start_block}) do - where(query, [transaction], transaction.block_number >= ^start_block) + defp where_start_transaction_block_match(query, %{start_block: start_block} = params) do + if DenormalizationHelper.denormalization_finished?() do + where(query, [transaction], transaction.block_number >= ^start_block) + else + where_start_block_match(query, params) + end end defp where_end_transaction_block_match(query, %{end_block: nil}), do: query - defp where_end_transaction_block_match(query, %{end_block: end_block}) do - where(query, [transaction], transaction.block_number <= ^end_block) + defp where_end_transaction_block_match(query, %{end_block: end_block} = params) do + if DenormalizationHelper.denormalization_finished?() do + where(query, [transaction], transaction.block_number <= ^end_block) + else + where_end_block_match(query, params) + end end defp where_start_timestamp_match(query, %{start_timestamp: nil}), do: query defp where_start_timestamp_match(query, %{start_timestamp: start_timestamp}) do - where(query, [transaction, _block], ^start_timestamp <= transaction.block_timestamp) + if DenormalizationHelper.denormalization_finished?() do + where(query, [transaction], ^start_timestamp <= transaction.block_timestamp) + else + where(query, [..., block], ^start_timestamp <= block.timestamp) + end end defp where_end_timestamp_match(query, %{end_timestamp: nil}), do: query defp where_end_timestamp_match(query, %{end_timestamp: end_timestamp}) do - where(query, [transaction, _block], transaction.block_timestamp <= ^end_timestamp) + if DenormalizationHelper.denormalization_finished?() do + where(query, [transaction], transaction.block_timestamp <= ^end_timestamp) + else + where(query, [..., block], block.timestamp <= ^end_timestamp) + end end defp where_contract_address_match(query, nil), do: query diff --git a/apps/explorer/lib/explorer/etherscan/logs.ex b/apps/explorer/lib/explorer/etherscan/logs.ex index 70c4dedaf2..e83587869d 100644 --- a/apps/explorer/lib/explorer/etherscan/logs.ex +++ b/apps/explorer/lib/explorer/etherscan/logs.ex @@ -5,10 +5,10 @@ defmodule Explorer.Etherscan.Logs do """ - import Ecto.Query, only: [from: 2, limit: 2, where: 3, subquery: 1, order_by: 3] + import Ecto.Query, only: [from: 2, limit: 2, where: 3, subquery: 1, order_by: 3, union: 2] - alias Explorer.Repo - alias Explorer.Chain.{Log, Transaction} + alias Explorer.{Chain, Repo} + alias Explorer.Chain.{Block, DenormalizationHelper, InternalTransaction, Log, Transaction} @base_filter %{ from_block: nil, @@ -76,52 +76,126 @@ defmodule Explorer.Etherscan.Logs do paging_options = if is_nil(paging_options), do: @default_paging_options, else: paging_options prepared_filter = Map.merge(@base_filter, filter) - logs_query = - Log - |> where_topic_match(prepared_filter) - |> where([log], log.address_hash == ^address_hash) - |> where([log], log.block_number >= ^prepared_filter.from_block) - |> where([log], log.block_number <= ^prepared_filter.to_block) - |> limit(1000) - |> order_by([log], asc: log.block_number, asc: log.index) - |> page_logs(paging_options) + if DenormalizationHelper.denormalization_finished?() do + logs_query = + Log + |> where_topic_match(prepared_filter) + |> where([log], log.address_hash == ^address_hash) + |> where([log], log.block_number >= ^prepared_filter.from_block) + |> where([log], log.block_number <= ^prepared_filter.to_block) + |> limit(1000) + |> order_by([log], asc: log.block_number, asc: log.index) + |> page_logs(paging_options) + + all_transaction_logs_query = + from(log in subquery(logs_query), + join: transaction in Transaction, + on: log.transaction_hash == transaction.hash, + select: map(log, ^@log_fields), + select_merge: %{ + gas_price: transaction.gas_price, + gas_used: transaction.gas_used, + transaction_index: transaction.index, + block_hash: transaction.block_hash, + block_number: transaction.block_number, + block_timestamp: transaction.block_timestamp, + block_consensus: transaction.block_consensus + } + ) - all_transaction_logs_query = - from(log in subquery(logs_query), - join: transaction in Transaction, - on: log.transaction_hash == transaction.hash, - select: map(log, ^@log_fields), - select_merge: %{ - gas_price: transaction.gas_price, - gas_used: transaction.gas_used, - transaction_index: transaction.index, - block_hash: transaction.block_hash, - block_number: transaction.block_number, - block_timestamp: transaction.block_timestamp, - block_consensus: transaction.block_consensus - } - ) + query_with_blocks = + from(log_transaction_data in subquery(all_transaction_logs_query), + where: log_transaction_data.address_hash == ^address_hash, + order_by: log_transaction_data.block_number, + select_merge: %{ + block_consensus: log_transaction_data.block_consensus + } + ) - query_with_blocks = - from(log_transaction_data in subquery(all_transaction_logs_query), - where: log_transaction_data.address_hash == ^address_hash, - order_by: log_transaction_data.block_number, - select_merge: %{ - block_consensus: log_transaction_data.block_consensus - } - ) + query_with_consensus = + if Map.get(filter, :allow_non_consensus) do + query_with_blocks + else + from([transaction] in query_with_blocks, + where: transaction.block_consensus == true + ) + end + + query_with_consensus + |> Repo.replica().all() + else + logs_query = where_topic_match(Log, prepared_filter) + + query_to_address_hash_wrapped = + logs_query + |> internal_transaction_query(:to_address_hash, prepared_filter, address_hash) + |> Chain.wrapped_union_subquery() + + query_from_address_hash_wrapped = + logs_query + |> internal_transaction_query(:from_address_hash, prepared_filter, address_hash) + |> Chain.wrapped_union_subquery() + + query_created_contract_address_hash_wrapped = + logs_query + |> internal_transaction_query(:created_contract_address_hash, prepared_filter, address_hash) + |> Chain.wrapped_union_subquery() + + internal_transaction_log_query = + query_to_address_hash_wrapped + |> union(^query_from_address_hash_wrapped) + |> union(^query_created_contract_address_hash_wrapped) + + all_transaction_logs_query = + from(transaction in Transaction, + join: log in ^logs_query, + on: log.transaction_hash == transaction.hash, + where: transaction.block_number >= ^prepared_filter.from_block, + where: transaction.block_number <= ^prepared_filter.to_block, + where: + transaction.to_address_hash == ^address_hash or + transaction.from_address_hash == ^address_hash or + transaction.created_contract_address_hash == ^address_hash, + select: map(log, ^@log_fields), + select_merge: %{ + gas_price: transaction.gas_price, + gas_used: transaction.gas_used, + transaction_index: transaction.index, + block_number: transaction.block_number + }, + union: ^internal_transaction_log_query + ) - query_with_consensus = - if Map.get(filter, :allow_non_consensus) do - query_with_blocks - else - from([transaction] in query_with_blocks, - where: transaction.block_consensus == true + query_with_blocks = + from(log_transaction_data in subquery(all_transaction_logs_query), + join: block in Block, + on: block.number == log_transaction_data.block_number, + where: log_transaction_data.address_hash == ^address_hash, + order_by: block.number, + limit: 1000, + select_merge: %{ + transaction_index: log_transaction_data.transaction_index, + block_hash: block.hash, + block_number: block.number, + block_timestamp: block.timestamp, + block_consensus: block.consensus + } ) - end - query_with_consensus - |> Repo.replica().all() + query_with_consensus = + if Map.get(filter, :allow_non_consensus) do + query_with_blocks + else + from([_, block] in query_with_blocks, + where: block.consensus == true + ) + end + + query_with_consensus + |> order_by([log], asc: log.index) + |> page_logs(paging_options) + |> Repo.replica().all() + end end # Since address_hash was not present, we know that a @@ -131,48 +205,90 @@ defmodule Explorer.Etherscan.Logs do def list_logs(filter, paging_options) do paging_options = if is_nil(paging_options), do: @default_paging_options, else: paging_options prepared_filter = Map.merge(@base_filter, filter) - logs_query = where_topic_match(Log, prepared_filter) - block_transaction_query = - from(transaction in Transaction, - where: transaction.block_number >= ^prepared_filter.from_block, - where: transaction.block_number <= ^prepared_filter.to_block, - select: %{ - transaction_hash: transaction.hash, - gas_price: transaction.gas_price, - gas_used: transaction.gas_used, - transaction_index: transaction.index, - block_hash: transaction.block_hash, - block_number: transaction.block_number, - block_timestamp: transaction.block_timestamp, - block_consensus: transaction.block_consensus - } - ) + if DenormalizationHelper.denormalization_finished?() do + block_transaction_query = + from(transaction in Transaction, + where: transaction.block_number >= ^prepared_filter.from_block, + where: transaction.block_number <= ^prepared_filter.to_block, + select: %{ + transaction_hash: transaction.hash, + gas_price: transaction.gas_price, + gas_used: transaction.gas_used, + transaction_index: transaction.index, + block_hash: transaction.block_hash, + block_number: transaction.block_number, + block_timestamp: transaction.block_timestamp, + block_consensus: transaction.block_consensus + } + ) - query_with_consensus = - if Map.get(filter, :allow_non_consensus) do - block_transaction_query - else - from([transaction] in block_transaction_query, - where: transaction.block_consensus == true + query_with_consensus = + if Map.get(filter, :allow_non_consensus) do + block_transaction_query + else + from([transaction] in block_transaction_query, + where: transaction.block_consensus == true + ) + end + + query_with_block_transaction_data = + from(log in logs_query, + join: block_transaction_data in subquery(query_with_consensus), + on: block_transaction_data.transaction_hash == log.transaction_hash, + order_by: block_transaction_data.block_number, + limit: 1000, + select: block_transaction_data, + select_merge: map(log, ^@log_fields) ) - end - - query_with_block_transaction_data = - from(log in logs_query, - join: block_transaction_data in subquery(query_with_consensus), - on: block_transaction_data.transaction_hash == log.transaction_hash, - order_by: block_transaction_data.block_number, - limit: 1000, - select: block_transaction_data, - select_merge: map(log, ^@log_fields) - ) - query_with_block_transaction_data - |> order_by([log], asc: log.index) - |> page_logs(paging_options) - |> Repo.replica().all() + query_with_block_transaction_data + |> order_by([log], asc: log.index) + |> page_logs(paging_options) + |> Repo.replica().all() + else + block_transaction_query = + from(transaction in Transaction, + join: block in assoc(transaction, :block), + where: block.number >= ^prepared_filter.from_block, + where: block.number <= ^prepared_filter.to_block, + select: %{ + transaction_hash: transaction.hash, + gas_price: transaction.gas_price, + gas_used: transaction.gas_used, + transaction_index: transaction.index, + block_hash: block.hash, + block_number: block.number, + block_timestamp: block.timestamp, + block_consensus: block.consensus + } + ) + + query_with_consensus = + if Map.get(filter, :allow_non_consensus) do + block_transaction_query + else + from([_, block] in block_transaction_query, + where: block.consensus == true + ) + end + + query_with_block_transaction_data = + from(log in logs_query, + join: block_transaction_data in subquery(query_with_consensus), + on: block_transaction_data.transaction_hash == log.transaction_hash, + order_by: block_transaction_data.block_number, + limit: 1000, + select: block_transaction_data, + select_merge: map(log, ^@log_fields) + ) + + query_with_block_transaction_data + |> order_by([log], asc: log.index) + |> page_logs(paging_options) + |> Repo.replica().all() + end end @topics [ @@ -232,4 +348,25 @@ defmodule Explorer.Etherscan.Logs do where: data.index > ^log_index and data.block_number >= ^block_number ) end + + defp internal_transaction_query(logs_query, direction, prepared_filter, address_hash) do + query = + from(internal_transaction in InternalTransaction.where_nonpending_block(), + join: transaction in assoc(internal_transaction, :transaction), + join: log in ^logs_query, + on: log.transaction_hash == internal_transaction.transaction_hash, + where: internal_transaction.block_number >= ^prepared_filter.from_block, + where: internal_transaction.block_number <= ^prepared_filter.to_block, + select: + merge(map(log, ^@log_fields), %{ + gas_price: transaction.gas_price, + gas_used: transaction.gas_used, + transaction_index: transaction.index, + block_number: internal_transaction.block_number + }) + ) + + query + |> InternalTransaction.where_address_fields_match(address_hash, direction) + end end diff --git a/apps/explorer/lib/explorer/transactions_denormalization_migrator.ex b/apps/explorer/lib/explorer/transactions_denormalization_migrator.ex new file mode 100644 index 0000000000..442344819c --- /dev/null +++ b/apps/explorer/lib/explorer/transactions_denormalization_migrator.ex @@ -0,0 +1,91 @@ +defmodule Explorer.TransactionsDenormalizationMigrator do + @moduledoc """ + Migrates all transactions to have set block_consensus and block_timestamp + """ + + use GenServer, restart: :transient + + import Ecto.Query + + alias Explorer.Chain.Cache.BackgroundMigrations + alias Explorer.Chain.Transaction + alias Explorer.Repo + + @default_batch_size 500 + @default_concurrency 4 * System.schedulers_online() + + @spec start_link(term()) :: GenServer.on_start() + def start_link(_) do + GenServer.start_link(__MODULE__, :ok, name: __MODULE__) + end + + def migration_finished? do + not Repo.exists?(unprocessed_transactions_query()) + end + + @impl true + def init(_) do + schedule_batch_migration() + {:ok, %{}} + end + + @impl true + def handle_info(:migrate_batch, state) do + case last_unprocessed_transaction_hashes() do + [] -> + BackgroundMigrations.set_denormalization_finished(true) + {:stop, :normal, state} + + hashes -> + hashes + |> Enum.chunk_every(batch_size()) + |> Enum.map(&run_task/1) + |> Task.await_many(:infinity) + + schedule_batch_migration() + + {:noreply, state} + end + end + + defp run_task(batch), do: Task.async(fn -> update_batch(batch) end) + + defp last_unprocessed_transaction_hashes do + limit = batch_size() * concurrency() + + unprocessed_transactions_query() + |> order_by(desc: :inserted_at) + |> select([t], t.hash) + |> limit(^limit) + |> Repo.all() + end + + defp unprocessed_transactions_query do + from(t in Transaction, + where: not is_nil(t.block_hash) and (is_nil(t.block_consensus) or is_nil(t.block_timestamp)) + ) + end + + defp update_batch(transaction_hashes) do + query = + from(transaction in Transaction, + join: block in assoc(transaction, :block), + where: transaction.hash in ^transaction_hashes, + update: [set: [block_consensus: block.consensus, block_timestamp: block.timestamp]] + ) + + Repo.update_all(query, [], timeout: :infinity) + end + + defp schedule_batch_migration do + Process.send(self(), :migrate_batch, []) + end + + defp batch_size do + Application.get_env(:explorer, __MODULE__)[:batch_size] || @default_batch_size + end + + defp concurrency do + Application.get_env(:explorer, __MODULE__)[:batch_size] || @default_concurrency + end +end diff --git a/apps/explorer/priv/repo/migrations/20220315082902_add_consensus_to_transaction_table.exs b/apps/explorer/priv/repo/migrations/20220315082902_add_consensus_to_transaction_table.exs index bb1c60a94d..8c8bbece02 100644 --- a/apps/explorer/priv/repo/migrations/20220315082902_add_consensus_to_transaction_table.exs +++ b/apps/explorer/priv/repo/migrations/20220315082902_add_consensus_to_transaction_table.exs @@ -6,14 +6,6 @@ defmodule Explorer.Repo.Migrations.AddConsensusToTransactionTable do add(:block_consensus, :boolean, default: true) end - execute(""" - UPDATE transactions tx - SET block_consensus = b.consensus - FROM blocks b - WHERE b.hash = tx.block_hash - AND b.consensus = false; - """) - create(index(:transactions, :block_consensus)) end end diff --git a/apps/explorer/priv/repo/migrations/20220315093927_add_block_timestamp_to_transaction_table.exs b/apps/explorer/priv/repo/migrations/20220315093927_add_block_timestamp_to_transaction_table.exs index 22f491027d..7303c9fce1 100644 --- a/apps/explorer/priv/repo/migrations/20220315093927_add_block_timestamp_to_transaction_table.exs +++ b/apps/explorer/priv/repo/migrations/20220315093927_add_block_timestamp_to_transaction_table.exs @@ -6,13 +6,6 @@ defmodule Explorer.Repo.Migrations.AddBlockTimestampToTransactionTable do add(:block_timestamp, :utc_datetime_usec) end - execute(""" - UPDATE transactions tx - SET block_timestamp = b.timestamp - FROM blocks b - WHERE b.hash = tx.block_hash; - """) - create(index(:transactions, :block_timestamp)) end end diff --git a/apps/explorer/test/explorer/transactions_denormalization_migrator_test.ex b/apps/explorer/test/explorer/transactions_denormalization_migrator_test.ex new file mode 100644 index 0000000000..12b9848370 --- /dev/null +++ b/apps/explorer/test/explorer/transactions_denormalization_migrator_test.ex @@ -0,0 +1,37 @@ +defmodule Explorer.TransactionsDenormalizationMigratorTest do + use Explorer.DataCase, async: false + + alias Explorer.Chain.Transaction + alias Explorer.{Repo, TransactionsDenormalizationMigrator} + + describe "Migrate transactions" do + test "Set block_consensus and block_timestamp for not processed transactions" do + Enum.each(0..10, fn _x -> + transaction = + :transaction + |> insert() + |> with_block(block_timestamp: nil, block_consensus: nil) + + assert %{block_consensus: nil, block_timestamp: nil, block: %{consensus: consensus, timestamp: timestamp}} = + transaction + + assert not is_nil(consensus) + assert not is_nil(timestamp) + end) + + TransactionsDenormalizationMigrator.start_link([]) + Process.sleep(100) + + Transaction + |> Repo.all() + |> Repo.preload(:block) + |> Enum.each(fn t -> + assert %{ + block_consensus: consensus, + block_timestamp: timestamp, + block: %{consensus: consensus, timestamp: timestamp} + } = t + end) + end + end +end diff --git a/apps/explorer/test/support/factory.ex b/apps/explorer/test/support/factory.ex index 0ca6f3ee06..4c218afb38 100644 --- a/apps/explorer/test/support/factory.ex +++ b/apps/explorer/test/support/factory.ex @@ -542,6 +542,8 @@ defmodule Explorer.Factory do cumulative_gas_used = collated_params[:cumulative_gas_used] || Enum.random(21_000..100_000) gas_used = collated_params[:gas_used] || Enum.random(21_000..100_000) status = Keyword.get(collated_params, :status, Enum.random([:ok, :error])) + block_timestamp = Keyword.get(collated_params, :block_timestamp, timestamp) + block_consensus = Keyword.get(collated_params, :block_consensus, true) error = (status == :error && collated_params[:error]) || nil @@ -556,8 +558,8 @@ defmodule Explorer.Factory do gas_used: gas_used, index: next_transaction_index, status: status, - block_timestamp: timestamp, - block_consensus: true + block_timestamp: block_timestamp, + block_consensus: block_consensus }) |> Repo.update!() |> Repo.preload(:block) diff --git a/cspell.json b/cspell.json index 78b98a92d7..d19d333d44 100644 --- a/cspell.json +++ b/cspell.json @@ -127,6 +127,8 @@ "DELEGATECALL", "delegators", "demonitor", + "denormalization", + "Denormalization", "Denormalized", "descr", "describedby", From 7c7639d357acf4e9e6303d9b27b60e1c62a4263e Mon Sep 17 00:00:00 2001 From: Qwerty5Uiop Date: Mon, 4 Dec 2023 18:59:56 +0600 Subject: [PATCH 12/33] Clear cache in GA --- .github/workflows/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/config.yml b/.github/workflows/config.yml index bd487d8c2a..30c3fa8c8c 100644 --- a/.github/workflows/config.yml +++ b/.github/workflows/config.yml @@ -195,7 +195,7 @@ jobs: id: dialyzer-cache with: path: priv/plts - key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-${{ matrix.chain-type }}-dialyzer-mixlockhash_25-${{ hashFiles('mix.lock') }} + key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-${{ matrix.chain-type }}-dialyzer-mixlockhash_28-${{ hashFiles('mix.lock') }} restore-keys: | ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-${{ matrix.chain-type }}-dialyzer-" From 66a1b390544824a199fbb10e7ab5b00931af13f0 Mon Sep 17 00:00:00 2001 From: Qwerty5Uiop Date: Mon, 4 Dec 2023 21:19:01 +0600 Subject: [PATCH 13/33] Use denormalization in token transfers csv exporter --- .../chain/csv_export/address_token_transfer_csv_exporter.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/explorer/lib/explorer/chain/csv_export/address_token_transfer_csv_exporter.ex b/apps/explorer/lib/explorer/chain/csv_export/address_token_transfer_csv_exporter.ex index 4409d1475e..5a44e272a7 100644 --- a/apps/explorer/lib/explorer/chain/csv_export/address_token_transfer_csv_exporter.ex +++ b/apps/explorer/lib/explorer/chain/csv_export/address_token_transfer_csv_exporter.ex @@ -12,7 +12,7 @@ defmodule Explorer.Chain.CSVExport.AddressTokenTransferCsvExporter do ] alias Explorer.{Chain, PagingOptions, Repo} - alias Explorer.Chain.{Address, Hash, TokenTransfer, Transaction} + alias Explorer.Chain.{Address, DenormalizationHelper, Hash, TokenTransfer, Transaction} alias Explorer.Chain.CSVExport.Helper @paging_options %PagingOptions{page_size: Helper.limit(), asc_order: true} @@ -118,7 +118,7 @@ defmodule Explorer.Chain.CSVExport.AddressTokenTransferCsvExporter do query |> handle_token_transfer_paging_options(paging_options) - |> preload(transaction: :block) + |> preload(^DenormalizationHelper.extend_transaction_preload([:transaction])) |> preload(:token) |> Repo.all() end From 27d042fc8fcf1bb25fdbf806465faa1f7a3183e7 Mon Sep 17 00:00:00 2001 From: Qwerty5Uiop Date: Wed, 6 Dec 2023 13:14:54 +0600 Subject: [PATCH 14/33] Clear cache in GA --- .github/workflows/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/config.yml b/.github/workflows/config.yml index 30c3fa8c8c..bd487d8c2a 100644 --- a/.github/workflows/config.yml +++ b/.github/workflows/config.yml @@ -195,7 +195,7 @@ jobs: id: dialyzer-cache with: path: priv/plts - key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-${{ matrix.chain-type }}-dialyzer-mixlockhash_28-${{ hashFiles('mix.lock') }} + key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-${{ matrix.chain-type }}-dialyzer-mixlockhash_25-${{ hashFiles('mix.lock') }} restore-keys: | ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-${{ matrix.chain-type }}-dialyzer-" From 39a8d6094a9ed1b086e7f5045896417aa4574d7d Mon Sep 17 00:00:00 2001 From: Qwerty5Uiop Date: Tue, 12 Dec 2023 17:05:49 +0600 Subject: [PATCH 15/33] Denormalization improvements --- .dialyzer-ignore | 2 +- .../lib/ethereum_jsonrpc/block.ex | 4 +- .../lib/ethereum_jsonrpc/blocks.ex | 34 +-------- .../lib/ethereum_jsonrpc/transaction.ex | 75 +++++++++++-------- .../lib/ethereum_jsonrpc/transactions.ex | 4 +- apps/explorer/lib/explorer/chain.ex | 18 +++-- ...dress_internal_transaction_csv_exporter.ex | 0 .../address_token_transfer_csv_exporter.ex | 0 .../chain/address_transaction_csv_exporter.ex | 0 .../explorer/chain/denormalization_helper.ex | 4 +- .../lib/explorer/chain/transaction.ex | 2 +- .../transactions_denormalization_migrator.ex | 20 +++-- .../lib/explorer/utility/migration_status.ex | 31 ++++++++ ...902_add_consensus_to_transaction_table.exs | 11 --- ...d_block_timestamp_to_transaction_table.exs | 11 --- ...imestamp_and_consensus_to_transactions.exs | 13 ++++ ...0231212102127_create_migrations_status.exs | 12 +++ apps/explorer/test/explorer/chain_test.exs | 16 +++- .../indexer/block/catchup/fetcher_test.exs | 2 +- .../fetcher/internal_transaction_test.exs | 2 +- .../test/indexer/fetcher/uncle_block_test.exs | 2 +- config/runtime.exs | 4 + 22 files changed, 153 insertions(+), 114 deletions(-) delete mode 100644 apps/explorer/lib/explorer/chain/address_internal_transaction_csv_exporter.ex delete mode 100644 apps/explorer/lib/explorer/chain/address_token_transfer_csv_exporter.ex delete mode 100644 apps/explorer/lib/explorer/chain/address_transaction_csv_exporter.ex create mode 100644 apps/explorer/lib/explorer/utility/migration_status.ex delete mode 100644 apps/explorer/priv/repo/migrations/20220315082902_add_consensus_to_transaction_table.exs delete mode 100644 apps/explorer/priv/repo/migrations/20220315093927_add_block_timestamp_to_transaction_table.exs create mode 100644 apps/explorer/priv/repo/migrations/20231212101547_add_block_timestamp_and_consensus_to_transactions.exs create mode 100644 apps/explorer/priv/repo/migrations/20231212102127_create_migrations_status.exs diff --git a/.dialyzer-ignore b/.dialyzer-ignore index a4baec9225..79cfeb4505 100644 --- a/.dialyzer-ignore +++ b/.dialyzer-ignore @@ -23,4 +23,4 @@ lib/indexer/fetcher/zkevm/transaction_batch.ex:156 lib/indexer/fetcher/zkevm/transaction_batch.ex:252 lib/block_scout_web/views/api/v2/transaction_view.ex:431 lib/block_scout_web/views/api/v2/transaction_view.ex:472 -lib/explorer/chain/transaction.ex:167 +lib/explorer/chain/transaction.ex:169 diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/block.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/block.ex index d07606297e..a3a32ddc0c 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/block.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/block.ex @@ -788,8 +788,8 @@ defmodule EthereumJSONRPC.Block do {key, timestamp_to_datetime(timestamp)} end - defp entry_to_elixir({"transactions" = key, transactions}, _block) do - {key, Transactions.to_elixir(transactions)} + defp entry_to_elixir({"transactions" = key, transactions}, %{"timestamp" => block_timestamp}) do + {key, Transactions.to_elixir(transactions, timestamp_to_datetime(block_timestamp))} end defp entry_to_elixir({"withdrawals" = key, nil}, _block) do diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/blocks.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/blocks.ex index a504468f49..d9a697c1ac 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/blocks.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/blocks.ex @@ -54,13 +54,12 @@ defmodule EthereumJSONRPC.Blocks do transactions_params = Transactions.elixir_to_params(elixir_transactions) withdrawals_params = Withdrawals.elixir_to_params(elixir_withdrawals) blocks_params = elixir_to_params(elixir_blocks) - transactions_params_with_block_timestamp = add_timestamp_to_transactions_params(transactions_params, blocks_params) %__MODULE__{ errors: errors, blocks_params: blocks_params, block_second_degree_relations_params: block_second_degree_relations_params, - transactions_params: transactions_params_with_block_timestamp, + transactions_params: transactions_params, withdrawals_params: withdrawals_params } end @@ -455,35 +454,4 @@ defmodule EthereumJSONRPC.Blocks do def to_elixir(blocks) when is_list(blocks) do Enum.map(blocks, &Block.to_elixir/1) end - - defp add_timestamp_to_transactions_params(transactions_params, blocks_params) do - block_hashes = - transactions_params - |> Enum.map(fn %{block_hash: block_hash} -> block_hash end) - |> Enum.uniq() - - block_hash_timestamp_map = - block_hashes - |> Enum.map(fn block_hash -> - block = - Enum.find(blocks_params, fn block_param -> - block_param.hash == block_hash - end) - - %{} - |> Map.put("#{block_hash}", block.timestamp) - end) - |> Enum.reduce(%{}, fn hash_timestamp_map_item, acc -> - Map.merge(acc, hash_timestamp_map_item) - end) - - transactions_params - |> Enum.map(fn transactions_param -> - Map.put( - transactions_param, - :block_timestamp, - Map.get(block_hash_timestamp_map, "#{transactions_param.block_hash}") - ) - end) - end end diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/transaction.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/transaction.ex index 964dc2ec1f..2f11da8520 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/transaction.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/transaction.ex @@ -237,11 +237,10 @@ defmodule EthereumJSONRPC.Transaction do result end - if transaction["creates"] do - Map.put(result, :created_contract_address_hash, transaction["creates"]) - else - result - end + put_if_present(transaction, result, [ + {"creates", :created_contract_address_hash}, + {"block_timestamp", :block_timestamp} + ]) end def elixir_to_params( @@ -286,11 +285,10 @@ defmodule EthereumJSONRPC.Transaction do max_fee_per_gas: max_fee_per_gas } - if transaction["creates"] do - Map.put(result, :created_contract_address_hash, transaction["creates"]) - else - result - end + put_if_present(transaction, result, [ + {"creates", :created_contract_address_hash}, + {"block_timestamp", :block_timestamp} + ]) end # txpool_content method on Erigon node returns tx data @@ -336,11 +334,10 @@ defmodule EthereumJSONRPC.Transaction do max_fee_per_gas: max_fee_per_gas } - if transaction["creates"] do - Map.put(result, :created_contract_address_hash, transaction["creates"]) - else - result - end + put_if_present(transaction, result, [ + {"creates", :created_contract_address_hash}, + {"block_timestamp", :block_timestamp} + ]) end # this is for Suave chain (handles `executionNode` and `requestRecord` fields without EIP-1559 fields) @@ -407,11 +404,10 @@ defmodule EthereumJSONRPC.Transaction do result end - if transaction["creates"] do - Map.put(result, :created_contract_address_hash, transaction["creates"]) - else - result - end + put_if_present(transaction, result, [ + {"creates", :created_contract_address_hash}, + {"block_timestamp", :block_timestamp} + ]) end def elixir_to_params( @@ -452,11 +448,10 @@ defmodule EthereumJSONRPC.Transaction do type: type } - if transaction["creates"] do - Map.put(result, :created_contract_address_hash, transaction["creates"]) - else - result - end + put_if_present(transaction, result, [ + {"creates", :created_contract_address_hash}, + {"block_timestamp", :block_timestamp} + ]) end def elixir_to_params( @@ -495,11 +490,10 @@ defmodule EthereumJSONRPC.Transaction do transaction_index: index } - if transaction["creates"] do - Map.put(result, :created_contract_address_hash, transaction["creates"]) - else - result - end + put_if_present(transaction, result, [ + {"creates", :created_contract_address_hash}, + {"block_timestamp", :block_timestamp} + ]) end @doc """ @@ -580,11 +574,14 @@ defmodule EthereumJSONRPC.Transaction do } """ - def to_elixir(transaction) when is_map(transaction) do - Enum.into(transaction, %{}, &entry_to_elixir/1) + def to_elixir(transaction, block_timestamp \\ nil) + + def to_elixir(transaction, block_timestamp) when is_map(transaction) do + initial = (block_timestamp && %{"block_timestamp" => block_timestamp}) || %{} + Enum.into(transaction, initial, &entry_to_elixir/1) end - def to_elixir(transaction) when is_binary(transaction) do + def to_elixir(transaction, _block_timestamp) when is_binary(transaction) do nil end @@ -658,4 +655,16 @@ defmodule EthereumJSONRPC.Transaction do defp entry_to_elixir(_) do {nil, nil} end + + defp put_if_present(transaction, result, keys) do + Enum.reduce(keys, result, fn {from_key, to_key}, acc -> + value = transaction[from_key] + + if value do + Map.put(acc, to_key, value) + else + acc + end + end) + end end diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/transactions.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/transactions.ex index 9b39378739..ecdf103b4e 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/transactions.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/transactions.ex @@ -151,9 +151,9 @@ defmodule EthereumJSONRPC.Transactions do ] """ - def to_elixir(transactions) when is_list(transactions) do + def to_elixir(transactions, block_timestamp \\ nil) when is_list(transactions) do transactions - |> Enum.map(&Transaction.to_elixir/1) + |> Enum.map(&Transaction.to_elixir(&1, block_timestamp)) |> Enum.filter(&(!is_nil(&1))) end end diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index b6d27141d7..4e0e5ac0c3 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -539,13 +539,17 @@ defmodule Explorer.Chain do ) end - query - |> Repo.all() - |> (&if(Enum.count(&1) > 0, - do: &1, - else: Enum.zip([block_hashes, for(_ <- 1..Enum.count(block_hashes), do: %Wei{value: Decimal.new(0)})]) - )).() - |> Enum.into(%{}) + initial_gas_payments = + block_hashes + |> Enum.map(&{&1, %Wei{value: Decimal.new(0)}}) + |> Enum.into(%{}) + + existing_data = + query + |> Repo.all() + |> Enum.into(%{}) + + Map.merge(initial_gas_payments, existing_data) end def timestamp_by_block_hash(block_hashes) when is_list(block_hashes) do diff --git a/apps/explorer/lib/explorer/chain/address_internal_transaction_csv_exporter.ex b/apps/explorer/lib/explorer/chain/address_internal_transaction_csv_exporter.ex deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/apps/explorer/lib/explorer/chain/address_token_transfer_csv_exporter.ex b/apps/explorer/lib/explorer/chain/address_token_transfer_csv_exporter.ex deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/apps/explorer/lib/explorer/chain/address_transaction_csv_exporter.ex b/apps/explorer/lib/explorer/chain/address_transaction_csv_exporter.ex deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/apps/explorer/lib/explorer/chain/denormalization_helper.ex b/apps/explorer/lib/explorer/chain/denormalization_helper.ex index 284ccf64a2..79c9d4c51a 100644 --- a/apps/explorer/lib/explorer/chain/denormalization_helper.ex +++ b/apps/explorer/lib/explorer/chain/denormalization_helper.ex @@ -1,5 +1,7 @@ defmodule Explorer.Chain.DenormalizationHelper do - @moduledoc false + @moduledoc """ + Helper functions for dynamic logic based on denormalization migration completeness + """ alias Explorer.Chain.Cache.BackgroundMigrations diff --git a/apps/explorer/lib/explorer/chain/transaction.ex b/apps/explorer/lib/explorer/chain/transaction.ex index a77700db8e..10b730d6bc 100644 --- a/apps/explorer/lib/explorer/chain/transaction.ex +++ b/apps/explorer/lib/explorer/chain/transaction.ex @@ -553,10 +553,10 @@ defmodule Explorer.Chain.Transaction do |> unique_constraint(:hash) end + @spec block_timestamp(t()) :: DateTime.t() def block_timestamp(%{block_number: nil, inserted_at: time}), do: time def block_timestamp(%{block_timestamp: time}) when not is_nil(time), do: time def block_timestamp(%{block: %{timestamp: time}}), do: time - def block_timestamp(_), do: nil def preload_token_transfers(query, address_hash) do token_transfers_query = diff --git a/apps/explorer/lib/explorer/transactions_denormalization_migrator.ex b/apps/explorer/lib/explorer/transactions_denormalization_migrator.ex index 442344819c..55f927d5af 100644 --- a/apps/explorer/lib/explorer/transactions_denormalization_migrator.ex +++ b/apps/explorer/lib/explorer/transactions_denormalization_migrator.ex @@ -10,9 +10,10 @@ defmodule Explorer.TransactionsDenormalizationMigrator do alias Explorer.Chain.Cache.BackgroundMigrations alias Explorer.Chain.Transaction alias Explorer.Repo + alias Explorer.Utility.MigrationStatus @default_batch_size 500 - @default_concurrency 4 * System.schedulers_online() + @migration_name "denormalization" @spec start_link(term()) :: GenServer.on_start() def start_link(_) do @@ -25,8 +26,15 @@ defmodule Explorer.TransactionsDenormalizationMigrator do @impl true def init(_) do - schedule_batch_migration() - {:ok, %{}} + case MigrationStatus.get_status(@migration_name) do + "completed" -> + :ignore + + _ -> + MigrationStatus.set_status(@migration_name, "started") + schedule_batch_migration() + {:ok, %{}} + end end @impl true @@ -34,6 +42,7 @@ defmodule Explorer.TransactionsDenormalizationMigrator do case last_unprocessed_transaction_hashes() do [] -> BackgroundMigrations.set_denormalization_finished(true) + MigrationStatus.set_status(@migration_name, "completed") {:stop, :normal, state} hashes -> @@ -54,7 +63,6 @@ defmodule Explorer.TransactionsDenormalizationMigrator do limit = batch_size() * concurrency() unprocessed_transactions_query() - |> order_by(desc: :inserted_at) |> select([t], t.hash) |> limit(^limit) |> Repo.all() @@ -86,6 +94,8 @@ defmodule Explorer.TransactionsDenormalizationMigrator do end defp concurrency do - Application.get_env(:explorer, __MODULE__)[:batch_size] || @default_concurrency + default = 4 * System.schedulers_online() + + Application.get_env(:explorer, __MODULE__)[:concurrency] || default end end diff --git a/apps/explorer/lib/explorer/utility/migration_status.ex b/apps/explorer/lib/explorer/utility/migration_status.ex new file mode 100644 index 0000000000..21ac52de6d --- /dev/null +++ b/apps/explorer/lib/explorer/utility/migration_status.ex @@ -0,0 +1,31 @@ +defmodule Explorer.Utility.MigrationStatus do + @moduledoc """ + Module is responsible for keeping the current status of background migrations. + """ + use Explorer.Schema + + alias Explorer.Repo + + @primary_key false + schema "migrations_status" do + field(:migration_name, :string) + field(:status, :string) + + timestamps() + end + + @doc false + def changeset(migration_status \\ %__MODULE__{}, params) do + cast(migration_status, params, [:migration_name, :status]) + end + + def get_status(migration_name) do + Repo.one(from(ms in __MODULE__, where: ms.migration_name == ^migration_name, select: ms.status)) + end + + def set_status(migration_name, status) do + %{migration_name: migration_name, status: status} + |> changeset() + |> Repo.insert(on_conflict: :replace_all, conflict_target: :migration_name) + end +end diff --git a/apps/explorer/priv/repo/migrations/20220315082902_add_consensus_to_transaction_table.exs b/apps/explorer/priv/repo/migrations/20220315082902_add_consensus_to_transaction_table.exs deleted file mode 100644 index 8c8bbece02..0000000000 --- a/apps/explorer/priv/repo/migrations/20220315082902_add_consensus_to_transaction_table.exs +++ /dev/null @@ -1,11 +0,0 @@ -defmodule Explorer.Repo.Migrations.AddConsensusToTransactionTable do - use Ecto.Migration - - def change do - alter table("transactions") do - add(:block_consensus, :boolean, default: true) - end - - create(index(:transactions, :block_consensus)) - end -end diff --git a/apps/explorer/priv/repo/migrations/20220315093927_add_block_timestamp_to_transaction_table.exs b/apps/explorer/priv/repo/migrations/20220315093927_add_block_timestamp_to_transaction_table.exs deleted file mode 100644 index 7303c9fce1..0000000000 --- a/apps/explorer/priv/repo/migrations/20220315093927_add_block_timestamp_to_transaction_table.exs +++ /dev/null @@ -1,11 +0,0 @@ -defmodule Explorer.Repo.Migrations.AddBlockTimestampToTransactionTable do - use Ecto.Migration - - def change do - alter table("transactions") do - add(:block_timestamp, :utc_datetime_usec) - end - - create(index(:transactions, :block_timestamp)) - end -end diff --git a/apps/explorer/priv/repo/migrations/20231212101547_add_block_timestamp_and_consensus_to_transactions.exs b/apps/explorer/priv/repo/migrations/20231212101547_add_block_timestamp_and_consensus_to_transactions.exs new file mode 100644 index 0000000000..458e296043 --- /dev/null +++ b/apps/explorer/priv/repo/migrations/20231212101547_add_block_timestamp_and_consensus_to_transactions.exs @@ -0,0 +1,13 @@ +defmodule Explorer.Repo.Migrations.AddBlockTimestampAndConsensusToTransactions do + use Ecto.Migration + + def change do + alter table(:transactions) do + add_if_not_exists(:block_timestamp, :utc_datetime_usec) + add_if_not_exists(:block_consensus, :boolean, default: true) + end + + create_if_not_exists(index(:transactions, :block_timestamp)) + create_if_not_exists(index(:transactions, :block_consensus)) + end +end diff --git a/apps/explorer/priv/repo/migrations/20231212102127_create_migrations_status.exs b/apps/explorer/priv/repo/migrations/20231212102127_create_migrations_status.exs new file mode 100644 index 0000000000..0b8bf54a5c --- /dev/null +++ b/apps/explorer/priv/repo/migrations/20231212102127_create_migrations_status.exs @@ -0,0 +1,12 @@ +defmodule Explorer.Repo.Migrations.CreateMigrationsStatus do + use Ecto.Migration + + def change do + create table(:migrations_status, primary_key: false) do + add(:migration_name, :string, primary_key: true) + add(:status, :string) + + timestamps() + end + end +end diff --git a/apps/explorer/test/explorer/chain_test.exs b/apps/explorer/test/explorer/chain_test.exs index 739e051ac5..2bbbaebcd2 100644 --- a/apps/explorer/test/explorer/chain_test.exs +++ b/apps/explorer/test/explorer/chain_test.exs @@ -869,7 +869,9 @@ defmodule Explorer.ChainTest do test "returns the correct address if it exists" do address = insert(:address) - assert {:ok, _address} = Chain.hash_to_address(address.hash) + assert {:ok, address_from_db} = Chain.hash_to_address(address.hash) + assert address_from_db.hash == address.hash + assert address_from_db.inserted_at == address.inserted_at end test "has_decompiled_code? is true if there are decompiled contracts" do @@ -918,14 +920,16 @@ defmodule Explorer.ChainTest do test "returns an address if it already exists" do address = insert(:address) - assert {:ok, _address} = Chain.find_or_insert_address_from_hash(address.hash) + assert {:ok, address_from_db} = Chain.find_or_insert_address_from_hash(address.hash) + assert address_from_db.hash == address.hash + assert address_from_db.inserted_at == address.inserted_at end test "returns an address if it doesn't exist" do hash_str = "0xcbbcd5ac86f9a50e13313633b262e16f695a90c2" {:ok, hash} = Chain.string_to_address_hash(hash_str) - assert {:ok, %Chain.Address{hash: _hash}} = Chain.find_or_insert_address_from_hash(hash) + assert {:ok, %Chain.Address{hash: ^hash}} = Chain.find_or_insert_address_from_hash(hash) end end @@ -3984,7 +3988,11 @@ defmodule Explorer.ChainTest do assert {:ok, result} = Chain.token_from_address_hash(token.contract_address_hash, options) - assert result.contract_address.smart_contract + assert address.smart_contract.address_hash == result.contract_address.smart_contract.address_hash + assert address.smart_contract.contract_code_md5 == result.contract_address.smart_contract.contract_code_md5 + assert address.smart_contract.abi == result.contract_address.smart_contract.abi + assert address.smart_contract.contract_source_code == result.contract_address.smart_contract.contract_source_code + assert address.smart_contract.name == result.contract_address.smart_contract.name end end diff --git a/apps/indexer/test/indexer/block/catchup/fetcher_test.exs b/apps/indexer/test/indexer/block/catchup/fetcher_test.exs index 4c6c6fdff0..93f687907c 100644 --- a/apps/indexer/test/indexer/block/catchup/fetcher_test.exs +++ b/apps/indexer/test/indexer/block/catchup/fetcher_test.exs @@ -456,7 +456,7 @@ defmodule Indexer.Block.Catchup.FetcherTest do assert count(Chain.Block) == 1 assert count(Reward) == 0 - assert_receive {:block_numbers, [_block_number]}, 5_000 + assert_receive {:block_numbers, [^block_number]}, 5_000 end test "async fetches beneficiaries when entire call errors out", %{ diff --git a/apps/indexer/test/indexer/fetcher/internal_transaction_test.exs b/apps/indexer/test/indexer/fetcher/internal_transaction_test.exs index f1f510c8ac..e451525869 100644 --- a/apps/indexer/test/indexer/fetcher/internal_transaction_test.exs +++ b/apps/indexer/test/indexer/fetcher/internal_transaction_test.exs @@ -305,7 +305,7 @@ defmodule Indexer.Fetcher.InternalTransactionTest do assert {:retry, [block.number]} == InternalTransaction.run([block.number, block.number], json_rpc_named_arguments) - assert %{block_hash: _block_hash} = Repo.get(PendingBlockOperation, block_hash) + assert %{block_hash: ^block_hash} = Repo.get(PendingBlockOperation, block_hash) end test "remove block consensus on foreign_key_violation", %{ diff --git a/apps/indexer/test/indexer/fetcher/uncle_block_test.exs b/apps/indexer/test/indexer/fetcher/uncle_block_test.exs index d5e27c9d76..052b4b75f4 100644 --- a/apps/indexer/test/indexer/fetcher/uncle_block_test.exs +++ b/apps/indexer/test/indexer/fetcher/uncle_block_test.exs @@ -196,7 +196,7 @@ defmodule Indexer.Fetcher.UncleBlockTest do ]} end) - assert {:retry, [_entry]} = + assert {:retry, [^entry]} = UncleBlock.run(entries, %Block.Fetcher{json_rpc_named_arguments: json_rpc_named_arguments}) end end diff --git a/config/runtime.exs b/config/runtime.exs index afcd147129..9c710d4ac1 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -456,6 +456,10 @@ config :explorer, Explorer.MicroserviceInterfaces.BENS, service_url: System.get_env("MICROSERVICE_BENS_URL"), enabled: ConfigHelper.parse_bool_env_var("MICROSERVICE_BENS_ENABLED") +config :explorer, Explorer.TransactionsDenormalizationMigrator, + batch_size: ConfigHelper.parse_integer_env_var("DENORMALIZATION_MIGRATION_BATCH_SIZE", 500), + concurrency: ConfigHelper.parse_integer_env_var("DENORMALIZATION_MIGRATION_CONCURRENCY", 10) + ############### ### Indexer ### ############### From efef76742a107e00134155d8a0829684834ee7f7 Mon Sep 17 00:00:00 2001 From: Qwerty5Uiop Date: Fri, 15 Dec 2023 15:01:15 +0600 Subject: [PATCH 16/33] Denormalization improvements --- .dialyzer-ignore | 2 +- apps/explorer/lib/explorer/chain.ex | 29 ++- .../explorer/chain/cache/gas_price_oracle.ex | 217 ++++++++++++------ .../explorer/chain/denormalization_helper.ex | 4 + .../lib/explorer/utility/migration_status.ex | 1 + ...actions_denormalization_migrator_test.exs} | 0 docker-compose/envs/common-blockscout.env | 4 +- 7 files changed, 175 insertions(+), 82 deletions(-) rename apps/explorer/test/explorer/{transactions_denormalization_migrator_test.ex => transactions_denormalization_migrator_test.exs} (100%) diff --git a/.dialyzer-ignore b/.dialyzer-ignore index 79cfeb4505..a9bc3a2c50 100644 --- a/.dialyzer-ignore +++ b/.dialyzer-ignore @@ -23,4 +23,4 @@ lib/indexer/fetcher/zkevm/transaction_batch.ex:156 lib/indexer/fetcher/zkevm/transaction_batch.ex:252 lib/block_scout_web/views/api/v2/transaction_view.ex:431 lib/block_scout_web/views/api/v2/transaction_view.ex:472 -lib/explorer/chain/transaction.ex:169 +lib/explorer/chain/transaction.ex:170 diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index 4e0e5ac0c3..19983d0045 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -353,15 +353,26 @@ defmodule Explorer.Chain do to_block = to_block(options) base = - from(log in Log, - order_by: [desc: log.block_number, desc: log.index], - where: log.address_hash == ^address_hash, - limit: ^paging_options.page_size, - select: log, - inner_join: block in Block, - on: block.hash == log.block_hash, - where: block.consensus == true - ) + if DenormalizationHelper.denormalization_finished?() do + from(log in Log, + order_by: [desc: log.block_number, desc: log.index], + where: log.address_hash == ^address_hash, + limit: ^paging_options.page_size, + select: log, + inner_join: transaction in assoc(log, :transaction), + where: transaction.block_consensus == true + ) + else + from(log in Log, + order_by: [desc: log.block_number, desc: log.index], + where: log.address_hash == ^address_hash, + limit: ^paging_options.page_size, + select: log, + inner_join: block in Block, + on: block.hash == log.block_hash, + where: block.consensus == true + ) + end preloaded_query = if csv_export? do diff --git a/apps/explorer/lib/explorer/chain/cache/gas_price_oracle.ex b/apps/explorer/lib/explorer/chain/cache/gas_price_oracle.ex index b1d0c10572..d6d85c2b42 100644 --- a/apps/explorer/lib/explorer/chain/cache/gas_price_oracle.ex +++ b/apps/explorer/lib/explorer/chain/cache/gas_price_oracle.ex @@ -14,6 +14,8 @@ defmodule Explorer.Chain.Cache.GasPriceOracle do alias Explorer.Chain.{ Block, + DenormalizationHelper, + Transaction, Wei } @@ -73,77 +75,150 @@ defmodule Explorer.Chain.Cache.GasPriceOracle do end fee_query = - from( - block in Block, - left_join: transaction in assoc(block, :transactions), - where: block.consensus == true, - where: transaction.status == ^1, - where: transaction.gas_price > ^0, - where: transaction.block_number > ^from_block, - group_by: transaction.block_number, - order_by: [desc: transaction.block_number], - select: %{ - block_number: transaction.block_number, - slow_gas_price: - fragment( - "percentile_disc(? :: real) within group ( order by ? )", - ^safelow_percentile_fraction, - transaction.gas_price - ), - average_gas_price: - fragment( - "percentile_disc(? :: real) within group ( order by ? )", - ^average_percentile_fraction, - transaction.gas_price - ), - fast_gas_price: - fragment( - "percentile_disc(? :: real) within group ( order by ? )", - ^fast_percentile_fraction, - transaction.gas_price - ), - slow_priority_fee_per_gas: - fragment( - "percentile_disc(? :: real) within group ( order by ? )", - ^safelow_percentile_fraction, - transaction.max_priority_fee_per_gas - ), - average_priority_fee_per_gas: - fragment( - "percentile_disc(? :: real) within group ( order by ? )", - ^average_percentile_fraction, - transaction.max_priority_fee_per_gas - ), - fast_priority_fee_per_gas: - fragment( - "percentile_disc(? :: real) within group ( order by ? )", - ^fast_percentile_fraction, - transaction.max_priority_fee_per_gas - ), - slow_time: - fragment( - "percentile_disc(? :: real) within group ( order by coalesce(extract(milliseconds from (?)::interval), ?) desc )", - ^safelow_percentile_fraction, - block.timestamp - transaction.earliest_processing_start, - ^average_block_time - ), - average_time: - fragment( - "percentile_disc(? :: real) within group ( order by coalesce(extract(milliseconds from (?)::interval), ?) desc )", - ^average_percentile_fraction, - block.timestamp - transaction.earliest_processing_start, - ^average_block_time - ), - fast_time: - fragment( - "percentile_disc(? :: real) within group ( order by coalesce(extract(milliseconds from (?)::interval), ?) desc )", - ^fast_percentile_fraction, - block.timestamp - transaction.earliest_processing_start, - ^average_block_time - ) - }, - limit: ^num_of_blocks - ) + if DenormalizationHelper.denormalization_finished?() do + from( + transaction in Transaction, + where: transaction.block_consensus == true, + where: transaction.status == ^1, + where: transaction.gas_price > ^0, + where: transaction.block_number > ^from_block, + group_by: transaction.block_number, + order_by: [desc: transaction.block_number], + select: %{ + block_number: transaction.block_number, + slow_gas_price: + fragment( + "percentile_disc(? :: real) within group ( order by ? )", + ^safelow_percentile_fraction, + transaction.gas_price + ), + average_gas_price: + fragment( + "percentile_disc(? :: real) within group ( order by ? )", + ^average_percentile_fraction, + transaction.gas_price + ), + fast_gas_price: + fragment( + "percentile_disc(? :: real) within group ( order by ? )", + ^fast_percentile_fraction, + transaction.gas_price + ), + slow_priority_fee_per_gas: + fragment( + "percentile_disc(? :: real) within group ( order by ? )", + ^safelow_percentile_fraction, + transaction.max_priority_fee_per_gas + ), + average_priority_fee_per_gas: + fragment( + "percentile_disc(? :: real) within group ( order by ? )", + ^average_percentile_fraction, + transaction.max_priority_fee_per_gas + ), + fast_priority_fee_per_gas: + fragment( + "percentile_disc(? :: real) within group ( order by ? )", + ^fast_percentile_fraction, + transaction.max_priority_fee_per_gas + ), + slow_time: + fragment( + "percentile_disc(? :: real) within group ( order by coalesce(extract(milliseconds from (?)::interval), ?) desc )", + ^safelow_percentile_fraction, + transaction.block_timestamp - transaction.earliest_processing_start, + ^average_block_time + ), + average_time: + fragment( + "percentile_disc(? :: real) within group ( order by coalesce(extract(milliseconds from (?)::interval), ?) desc )", + ^average_percentile_fraction, + transaction.block_timestamp - transaction.earliest_processing_start, + ^average_block_time + ), + fast_time: + fragment( + "percentile_disc(? :: real) within group ( order by coalesce(extract(milliseconds from (?)::interval), ?) desc )", + ^fast_percentile_fraction, + transaction.block_timestamp - transaction.earliest_processing_start, + ^average_block_time + ) + }, + limit: ^num_of_blocks + ) + else + from( + block in Block, + left_join: transaction in assoc(block, :transactions), + where: block.consensus == true, + where: transaction.status == ^1, + where: transaction.gas_price > ^0, + where: transaction.block_number > ^from_block, + group_by: transaction.block_number, + order_by: [desc: transaction.block_number], + select: %{ + block_number: transaction.block_number, + slow_gas_price: + fragment( + "percentile_disc(? :: real) within group ( order by ? )", + ^safelow_percentile_fraction, + transaction.gas_price + ), + average_gas_price: + fragment( + "percentile_disc(? :: real) within group ( order by ? )", + ^average_percentile_fraction, + transaction.gas_price + ), + fast_gas_price: + fragment( + "percentile_disc(? :: real) within group ( order by ? )", + ^fast_percentile_fraction, + transaction.gas_price + ), + slow_priority_fee_per_gas: + fragment( + "percentile_disc(? :: real) within group ( order by ? )", + ^safelow_percentile_fraction, + transaction.max_priority_fee_per_gas + ), + average_priority_fee_per_gas: + fragment( + "percentile_disc(? :: real) within group ( order by ? )", + ^average_percentile_fraction, + transaction.max_priority_fee_per_gas + ), + fast_priority_fee_per_gas: + fragment( + "percentile_disc(? :: real) within group ( order by ? )", + ^fast_percentile_fraction, + transaction.max_priority_fee_per_gas + ), + slow_time: + fragment( + "percentile_disc(? :: real) within group ( order by coalesce(extract(milliseconds from (?)::interval), ?) desc )", + ^safelow_percentile_fraction, + block.timestamp - transaction.earliest_processing_start, + ^average_block_time + ), + average_time: + fragment( + "percentile_disc(? :: real) within group ( order by coalesce(extract(milliseconds from (?)::interval), ?) desc )", + ^average_percentile_fraction, + block.timestamp - transaction.earliest_processing_start, + ^average_block_time + ), + fast_time: + fragment( + "percentile_disc(? :: real) within group ( order by coalesce(extract(milliseconds from (?)::interval), ?) desc )", + ^fast_percentile_fraction, + block.timestamp - transaction.earliest_processing_start, + ^average_block_time + ) + }, + limit: ^num_of_blocks + ) + end new_acc = fee_query |> Repo.all(timeout: :infinity) |> merge_gas_prices(acc, num_of_blocks) diff --git a/apps/explorer/lib/explorer/chain/denormalization_helper.ex b/apps/explorer/lib/explorer/chain/denormalization_helper.ex index 79c9d4c51a..0199fc7359 100644 --- a/apps/explorer/lib/explorer/chain/denormalization_helper.ex +++ b/apps/explorer/lib/explorer/chain/denormalization_helper.ex @@ -5,6 +5,7 @@ defmodule Explorer.Chain.DenormalizationHelper do alias Explorer.Chain.Cache.BackgroundMigrations + @spec extend_block_necessity(keyword(), :optional | :required) :: keyword() def extend_block_necessity(opts, necessity \\ :optional) do if denormalization_finished?() do opts @@ -13,6 +14,7 @@ defmodule Explorer.Chain.DenormalizationHelper do end end + @spec extend_transaction_block_necessity(keyword(), :optional | :required) :: keyword() def extend_transaction_block_necessity(opts, necessity \\ :optional) do if denormalization_finished?() do opts @@ -26,6 +28,7 @@ defmodule Explorer.Chain.DenormalizationHelper do end end + @spec extend_transaction_preload(list()) :: list() def extend_transaction_preload(preloads) do if denormalization_finished?() do preloads @@ -34,6 +37,7 @@ defmodule Explorer.Chain.DenormalizationHelper do end end + @spec extend_block_preload(list()) :: list() def extend_block_preload(preloads) do if denormalization_finished?() do preloads diff --git a/apps/explorer/lib/explorer/utility/migration_status.ex b/apps/explorer/lib/explorer/utility/migration_status.ex index 21ac52de6d..62495541be 100644 --- a/apps/explorer/lib/explorer/utility/migration_status.ex +++ b/apps/explorer/lib/explorer/utility/migration_status.ex @@ -9,6 +9,7 @@ defmodule Explorer.Utility.MigrationStatus do @primary_key false schema "migrations_status" do field(:migration_name, :string) + # ["started", "completed"] field(:status, :string) timestamps() diff --git a/apps/explorer/test/explorer/transactions_denormalization_migrator_test.ex b/apps/explorer/test/explorer/transactions_denormalization_migrator_test.exs similarity index 100% rename from apps/explorer/test/explorer/transactions_denormalization_migrator_test.ex rename to apps/explorer/test/explorer/transactions_denormalization_migrator_test.exs diff --git a/docker-compose/envs/common-blockscout.env b/docker-compose/envs/common-blockscout.env index 07a2a81269..f1743f7226 100644 --- a/docker-compose/envs/common-blockscout.env +++ b/docker-compose/envs/common-blockscout.env @@ -267,4 +267,6 @@ API_V2_ENABLED=true # ACCOUNT_PRIVATE_TAGS_LIMIT=2000 # ACCOUNT_WATCHLIST_ADDRESSES_LIMIT=15 # MICROSERVICE_BENS_URL= -# MICROSERVICE_BENS_ENABLED= \ No newline at end of file +# MICROSERVICE_BENS_ENABLED= +# DENORMALIZATION_MIGRATION_BATCH_SIZE= +# DENORMALIZATION_MIGRATION_CONCURRENCY= From 5c05a7e0bf846b9ead50fb20c3b7c05953ac1e61 Mon Sep 17 00:00:00 2001 From: Qwerty5Uiop Date: Mon, 18 Dec 2023 11:29:13 +0600 Subject: [PATCH 17/33] Move denormalization migration modules to a separate dir --- apps/explorer/config/config.exs | 2 +- apps/explorer/config/runtime/test.exs | 2 +- apps/explorer/lib/explorer/application.ex | 2 +- .../lib/explorer/chain/cache/background_migrations.ex | 4 ++-- .../lib/explorer/{utility => migrator}/migration_status.ex | 2 +- .../transactions_denormalization.ex} | 4 ++-- .../transactions_denormalization_migrator_test.exs | 7 ++++--- config/runtime.exs | 2 +- 8 files changed, 13 insertions(+), 12 deletions(-) rename apps/explorer/lib/explorer/{utility => migrator}/migration_status.ex (94%) rename apps/explorer/lib/explorer/{transactions_denormalization_migrator.ex => migrator/transactions_denormalization.ex} (96%) rename apps/explorer/test/explorer/{ => migrator}/transactions_denormalization_migrator_test.exs (83%) diff --git a/apps/explorer/config/config.exs b/apps/explorer/config/config.exs index 88d38c68a9..1dac8ff096 100644 --- a/apps/explorer/config/config.exs +++ b/apps/explorer/config/config.exs @@ -113,7 +113,7 @@ config :explorer, Explorer.TokenTransferTokenIdMigration.Supervisor, enabled: tr config :explorer, Explorer.TokenInstanceOwnerAddressMigration.Supervisor, enabled: true -config :explorer, Explorer.TransactionsDenormalizationMigrator, enabled: true +config :explorer, Explorer.Migrator.TransactionsDenormalization, enabled: true config :explorer, Explorer.Chain.Fetcher.CheckBytecodeMatchingOnDemand, enabled: true diff --git a/apps/explorer/config/runtime/test.exs b/apps/explorer/config/runtime/test.exs index 37ce93b2cb..367f4793f2 100644 --- a/apps/explorer/config/runtime/test.exs +++ b/apps/explorer/config/runtime/test.exs @@ -37,7 +37,7 @@ config :explorer, Explorer.TokenTransferTokenIdMigration.Supervisor, enabled: fa config :explorer, Explorer.TokenInstanceOwnerAddressMigration.Supervisor, enabled: false -config :explorer, Explorer.TransactionsDenormalizationMigrator, enabled: false +config :explorer, Explorer.Migrator.TransactionsDenormalization, enabled: false config :explorer, realtime_events_sender: Explorer.Chain.Events.SimpleSender diff --git a/apps/explorer/lib/explorer/application.ex b/apps/explorer/lib/explorer/application.ex index 497043d143..dfed4625c8 100644 --- a/apps/explorer/lib/explorer/application.ex +++ b/apps/explorer/lib/explorer/application.ex @@ -128,7 +128,7 @@ defmodule Explorer.Application do configure(Explorer.TokenInstanceOwnerAddressMigration.Supervisor), sc_microservice_configure(Explorer.Chain.Fetcher.LookUpSmartContractSourcesOnDemand), configure(Explorer.Chain.Cache.RootstockLockedBTC), - configure(Explorer.TransactionsDenormalizationMigrator) + configure(Explorer.Migrator.TransactionsDenormalization) ] |> List.flatten() diff --git a/apps/explorer/lib/explorer/chain/cache/background_migrations.ex b/apps/explorer/lib/explorer/chain/cache/background_migrations.ex index 4b33076a96..5e3b352457 100644 --- a/apps/explorer/lib/explorer/chain/cache/background_migrations.ex +++ b/apps/explorer/lib/explorer/chain/cache/background_migrations.ex @@ -11,11 +11,11 @@ defmodule Explorer.Chain.Cache.BackgroundMigrations do @dialyzer :no_match - alias Explorer.TransactionsDenormalizationMigrator + alias Explorer.Migrator.TransactionsDenormalization defp handle_fallback(:denormalization_finished) do Task.start(fn -> - set_denormalization_finished(TransactionsDenormalizationMigrator.migration_finished?()) + set_denormalization_finished(TransactionsDenormalization.migration_finished?()) end) {:return, false} diff --git a/apps/explorer/lib/explorer/utility/migration_status.ex b/apps/explorer/lib/explorer/migrator/migration_status.ex similarity index 94% rename from apps/explorer/lib/explorer/utility/migration_status.ex rename to apps/explorer/lib/explorer/migrator/migration_status.ex index 62495541be..295a606f96 100644 --- a/apps/explorer/lib/explorer/utility/migration_status.ex +++ b/apps/explorer/lib/explorer/migrator/migration_status.ex @@ -1,4 +1,4 @@ -defmodule Explorer.Utility.MigrationStatus do +defmodule Explorer.Migrator.MigrationStatus do @moduledoc """ Module is responsible for keeping the current status of background migrations. """ diff --git a/apps/explorer/lib/explorer/transactions_denormalization_migrator.ex b/apps/explorer/lib/explorer/migrator/transactions_denormalization.ex similarity index 96% rename from apps/explorer/lib/explorer/transactions_denormalization_migrator.ex rename to apps/explorer/lib/explorer/migrator/transactions_denormalization.ex index 55f927d5af..2cdfd374ca 100644 --- a/apps/explorer/lib/explorer/transactions_denormalization_migrator.ex +++ b/apps/explorer/lib/explorer/migrator/transactions_denormalization.ex @@ -1,4 +1,4 @@ -defmodule Explorer.TransactionsDenormalizationMigrator do +defmodule Explorer.Migrator.TransactionsDenormalization do @moduledoc """ Migrates all transactions to have set block_consensus and block_timestamp """ @@ -9,8 +9,8 @@ defmodule Explorer.TransactionsDenormalizationMigrator do alias Explorer.Chain.Cache.BackgroundMigrations alias Explorer.Chain.Transaction + alias Explorer.Migrator.MigrationStatus alias Explorer.Repo - alias Explorer.Utility.MigrationStatus @default_batch_size 500 @migration_name "denormalization" diff --git a/apps/explorer/test/explorer/transactions_denormalization_migrator_test.exs b/apps/explorer/test/explorer/migrator/transactions_denormalization_migrator_test.exs similarity index 83% rename from apps/explorer/test/explorer/transactions_denormalization_migrator_test.exs rename to apps/explorer/test/explorer/migrator/transactions_denormalization_migrator_test.exs index 12b9848370..8505844a79 100644 --- a/apps/explorer/test/explorer/transactions_denormalization_migrator_test.exs +++ b/apps/explorer/test/explorer/migrator/transactions_denormalization_migrator_test.exs @@ -1,8 +1,9 @@ -defmodule Explorer.TransactionsDenormalizationMigratorTest do +defmodule Explorer.Migrator.TransactionsDenormalizationTest do use Explorer.DataCase, async: false alias Explorer.Chain.Transaction - alias Explorer.{Repo, TransactionsDenormalizationMigrator} + alias Explorer.Migrator.TransactionsDenormalization + alias Explorer.Repo describe "Migrate transactions" do test "Set block_consensus and block_timestamp for not processed transactions" do @@ -19,7 +20,7 @@ defmodule Explorer.TransactionsDenormalizationMigratorTest do assert not is_nil(timestamp) end) - TransactionsDenormalizationMigrator.start_link([]) + TransactionsDenormalization.start_link([]) Process.sleep(100) Transaction diff --git a/config/runtime.exs b/config/runtime.exs index 9c710d4ac1..e8f4de4496 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -456,7 +456,7 @@ config :explorer, Explorer.MicroserviceInterfaces.BENS, service_url: System.get_env("MICROSERVICE_BENS_URL"), enabled: ConfigHelper.parse_bool_env_var("MICROSERVICE_BENS_ENABLED") -config :explorer, Explorer.TransactionsDenormalizationMigrator, +config :explorer, Explorer.Migrator.TransactionsDenormalization, batch_size: ConfigHelper.parse_integer_env_var("DENORMALIZATION_MIGRATION_BATCH_SIZE", 500), concurrency: ConfigHelper.parse_integer_env_var("DENORMALIZATION_MIGRATION_CONCURRENCY", 10) From 62eb8da2d316eba3a9cdba0c57b247da22f5fe3f Mon Sep 17 00:00:00 2001 From: Qwerty5Uiop Date: Mon, 18 Dec 2023 14:38:15 +0600 Subject: [PATCH 18/33] Fix migration status upsert --- apps/explorer/lib/explorer/migrator/migration_status.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/explorer/lib/explorer/migrator/migration_status.ex b/apps/explorer/lib/explorer/migrator/migration_status.ex index 295a606f96..01a7bbd540 100644 --- a/apps/explorer/lib/explorer/migrator/migration_status.ex +++ b/apps/explorer/lib/explorer/migrator/migration_status.ex @@ -27,6 +27,6 @@ defmodule Explorer.Migrator.MigrationStatus do def set_status(migration_name, status) do %{migration_name: migration_name, status: status} |> changeset() - |> Repo.insert(on_conflict: :replace_all, conflict_target: :migration_name) + |> Repo.insert(on_conflict: {:replace_all_except, [:inserted_at]}, conflict_target: :migration_name) end end From e52c187ff334c4ed3fa458e3f48b7d9e543b3c28 Mon Sep 17 00:00:00 2001 From: Qwerty5Uiop Date: Mon, 18 Dec 2023 17:36:31 +0600 Subject: [PATCH 19/33] Set timeout: :infinity for denaromalization migration --- .../lib/explorer/migrator/transactions_denormalization.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/explorer/lib/explorer/migrator/transactions_denormalization.ex b/apps/explorer/lib/explorer/migrator/transactions_denormalization.ex index 2cdfd374ca..166d7f8340 100644 --- a/apps/explorer/lib/explorer/migrator/transactions_denormalization.ex +++ b/apps/explorer/lib/explorer/migrator/transactions_denormalization.ex @@ -65,7 +65,7 @@ defmodule Explorer.Migrator.TransactionsDenormalization do unprocessed_transactions_query() |> select([t], t.hash) |> limit(^limit) - |> Repo.all() + |> Repo.all(timeout: :infinity) end defp unprocessed_transactions_query do From ed1f2479d3a7f67f7a0c10eddf25e137b02ceae6 Mon Sep 17 00:00:00 2001 From: Viktor Baranov Date: Thu, 21 Dec 2023 15:56:49 +0300 Subject: [PATCH 20/33] Update CHANGELOG entry --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d07488b208..dbc203b9dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ - [#9009](https://github.com/blockscout/blockscout/pull/9009) - Index for block refetch_needed - [#9006](https://github.com/blockscout/blockscout/pull/9006) - Drop unused indexes on address_current_token_balances table - [#8996](https://github.com/blockscout/blockscout/pull/8996) - Refine token transfers token ids index +- [#5322](https://github.com/blockscout/blockscout/pull/5322) - DB denormalization: block consensus and timestamp in transaction table ## Current @@ -101,7 +102,6 @@ - [#8956](https://github.com/blockscout/blockscout/pull/8956) - Refine docker-compose config structure - [#8911](https://github.com/blockscout/blockscout/pull/8911) - Set client_connection_check_interval for main Postgres DB in docker-compose setup -- [#5322](https://github.com/blockscout/blockscout/pull/5322) - DB denormalization: block consensus and timestamp in transaction table
Dependencies version bumps From 4df5df71fa9968823a6acb0ac5617b20b547c679 Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Thu, 21 Dec 2023 17:02:16 +0300 Subject: [PATCH 21/33] Change log topic type in the DB to bytea (#9000) * Change log topic type * Process review comment * Update apps/explorer/lib/explorer/chain/token_transfer.ex Co-authored-by: Maxim Filonov <53992153+sl1depengwyn@users.noreply.github.com> * Accept a new log type by TransactionAction and PolygonEdge modules * mix format * Fix merging conflicts * Update CHANGE LOG entry --------- Co-authored-by: Maxim Filonov <53992153+sl1depengwyn@users.noreply.github.com> Co-authored-by: POA <33550681+poa@users.noreply.github.com> --- .dialyzer-ignore | 8 +- CHANGELOG.md | 1 + .../channels/websocket_v2_test.exs | 26 ++-- .../api/rpc/eth_controller_test.exs | 68 +++++++--- .../api/rpc/logs_controller_test.exs | 59 ++++++--- .../api/rpc/transaction_controller_test.exs | 18 ++- .../api/v2/address_controller_test.exs | 10 +- .../api/v2/transaction_controller_test.exs | 11 +- apps/explorer/lib/explorer/chain.ex | 26 ---- apps/explorer/lib/explorer/chain/log.ex | 33 +++-- .../lib/explorer/chain/token_transfer.ex | 29 +++- ...213152332_alter_log_topic_columns_type.exs | 18 +++ .../address_log_csv_exporter_test.exs | 18 ++- .../test/explorer/chain/import_test.exs | 22 ++- .../explorer/test/explorer/chain/log_test.exs | 30 +++-- .../explorer/chain/smart_contract_test.exs | 3 +- .../explorer/chain/token_transfer_test.exs | 24 ++++ apps/explorer/test/explorer/chain_test.exs | 62 ++++----- .../test/explorer/etherscan/logs_test.exs | 125 ++++++++++-------- .../fetcher/polygon_edge/deposit_execute.ex | 8 +- .../fetcher/polygon_edge/withdrawal.ex | 8 +- apps/indexer/lib/indexer/helper.ex | 9 ++ .../temporary/uncataloged_token_transfers.ex | 4 +- .../indexer/transform/transaction_actions.ex | 40 ++++-- 24 files changed, 436 insertions(+), 224 deletions(-) create mode 100644 apps/explorer/priv/repo/migrations/20231213152332_alter_log_topic_columns_type.exs diff --git a/.dialyzer-ignore b/.dialyzer-ignore index a9bc3a2c50..ba3b24756c 100644 --- a/.dialyzer-ignore +++ b/.dialyzer-ignore @@ -14,10 +14,10 @@ lib/phoenix/router.ex:324 lib/phoenix/router.ex:402 lib/explorer/smart_contract/reader.ex:435 lib/indexer/fetcher/polygon_edge.ex:737 -lib/indexer/fetcher/polygon_edge/deposit_execute.ex:140 -lib/indexer/fetcher/polygon_edge/deposit_execute.ex:184 -lib/indexer/fetcher/polygon_edge/withdrawal.ex:160 -lib/indexer/fetcher/polygon_edge/withdrawal.ex:204 +lib/indexer/fetcher/polygon_edge/deposit_execute.ex:146 +lib/indexer/fetcher/polygon_edge/deposit_execute.ex:190 +lib/indexer/fetcher/polygon_edge/withdrawal.ex:166 +lib/indexer/fetcher/polygon_edge/withdrawal.ex:210 lib/indexer/fetcher/zkevm/transaction_batch.ex:116 lib/indexer/fetcher/zkevm/transaction_batch.ex:156 lib/indexer/fetcher/zkevm/transaction_batch.ex:252 diff --git a/CHANGELOG.md b/CHANGELOG.md index dbc203b9dc..f852bfdf77 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ - [#9009](https://github.com/blockscout/blockscout/pull/9009) - Index for block refetch_needed - [#9006](https://github.com/blockscout/blockscout/pull/9006) - Drop unused indexes on address_current_token_balances table +- [#9000](https://github.com/blockscout/blockscout/pull/9000) - Change log topic type in the DB to bytea - [#8996](https://github.com/blockscout/blockscout/pull/8996) - Refine token transfers token ids index - [#5322](https://github.com/blockscout/blockscout/pull/5322) - DB denormalization: block consensus and timestamp in transaction table diff --git a/apps/block_scout_web/test/block_scout_web/channels/websocket_v2_test.exs b/apps/block_scout_web/test/block_scout_web/channels/websocket_v2_test.exs index bfb0424424..2e06c53dbd 100644 --- a/apps/block_scout_web/test/block_scout_web/channels/websocket_v2_test.exs +++ b/apps/block_scout_web/test/block_scout_web/channels/websocket_v2_test.exs @@ -6,7 +6,15 @@ defmodule BlockScoutWeb.WebsocketV2Test do alias Explorer.Chain.{Address, Import, Token, TokenTransfer, Transaction} alias Explorer.Repo + @first_topic_hex_string "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef" + @second_topic_hex_string "0x000000000000000000000000e8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca" + @third_topic_hex_string "0x000000000000000000000000515c09c5bba1ed566b02a5b0599ec5d5d0aee73d" + describe "websocket v2" do + {:ok, first_topic} = Explorer.Chain.Hash.Full.cast(@first_topic_hex_string) + {:ok, second_topic} = Explorer.Chain.Hash.Full.cast(@second_topic_hex_string) + {:ok, third_topic} = Explorer.Chain.Hash.Full.cast(@third_topic_hex_string) + @import_data %{ blocks: %{ params: [ @@ -34,9 +42,9 @@ defmodule BlockScoutWeb.WebsocketV2Test do block_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd", address_hash: "0x8bf38d4764929064f2d4d3a56520a76ab3df415b", data: "0x0000000000000000000000000000000000000000000000000de0b6b3a7640000", - first_topic: "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", - second_topic: "0x000000000000000000000000e8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca", - third_topic: "0x000000000000000000000000515c09c5bba1ed566b02a5b0599ec5d5d0aee73d", + first_topic: first_topic, + second_topic: second_topic, + third_topic: third_topic, fourth_topic: nil, index: 0, transaction_hash: "0x53bd884872de3e488692881baeec262e7b95234d3965248c39fe992fffd433e5", @@ -46,9 +54,9 @@ defmodule BlockScoutWeb.WebsocketV2Test do block_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd", address_hash: "0x8bf38d4764929064f2d4d3a56520a76ab3df415b", data: "0x0000000000000000000000000000000000000000000000000de0b6b3a7640000", - first_topic: "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", - second_topic: "0x000000000000000000000000e8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca", - third_topic: "0x000000000000000000000000515c09c5bba1ed566b02a5b0599ec5d5d0aee73d", + first_topic: first_topic, + second_topic: second_topic, + third_topic: third_topic, fourth_topic: nil, index: 1, transaction_hash: "0x53bd884872de3e488692881baeec262e7b95234d3965248c39fe992fffd433e5", @@ -58,9 +66,9 @@ defmodule BlockScoutWeb.WebsocketV2Test do block_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd", address_hash: "0x8bf38d4764929064f2d4d3a56520a76ab3df415b", data: "0x0000000000000000000000000000000000000000000000000de0b6b3a7640000", - first_topic: "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", - second_topic: "0x000000000000000000000000e8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca", - third_topic: "0x000000000000000000000000515c09c5bba1ed566b02a5b0599ec5d5d0aee73d", + first_topic: first_topic, + second_topic: second_topic, + third_topic: third_topic, fourth_topic: nil, index: 2, transaction_hash: "0x53bd884872de3e488692881baeec262e7b95234d3965248c39fe992fffd433e5", diff --git a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/eth_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/eth_controller_test.exs index 4b68122fd2..b77a23a39a 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/eth_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/eth_controller_test.exs @@ -5,6 +5,12 @@ defmodule BlockScoutWeb.API.RPC.EthControllerTest do alias Explorer.Repo alias Indexer.Fetcher.CoinBalanceOnDemand + @first_topic_hex_string_1 "0x7fcf532c15f0a6db0bd6d0e038bea71d30d808c7d98cb3bf7268a95bf5081b65" + @first_topic_hex_string_2 "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef" + + @second_topic_hex_string_1 "0x00000000000000000000000098a9dc37d3650b5b30d6c12789b3881ee0b70c16" + @second_topic_hex_string_2 "0x000000000000000000000000e2680fd7cdbb04e9087a647ad4d023ef6c8fb4e2" + setup do mocked_json_rpc_named_arguments = [ transport: EthereumJSONRPC.Mox, @@ -27,6 +33,11 @@ defmodule BlockScoutWeb.API.RPC.EthControllerTest do defp params(api_params, params), do: Map.put(api_params, "params", params) + defp topic(topic_hex_string) do + {:ok, topic} = Explorer.Chain.Hash.Full.cast(topic_hex_string) + topic + end + describe "eth_get_logs" do setup do %{ @@ -108,10 +119,10 @@ defmodule BlockScoutWeb.API.RPC.EthControllerTest do address: address, transaction: transaction, data: "0x010101", - first_topic: "0x01" + first_topic: topic(@first_topic_hex_string_1) ) - params = params(api_params, [%{"address" => to_string(address.hash), "topics" => ["0x01"]}]) + params = params(api_params, [%{"address" => to_string(address.hash), "topics" => [@first_topic_hex_string_1]}]) assert response = conn @@ -134,7 +145,7 @@ defmodule BlockScoutWeb.API.RPC.EthControllerTest do block_number: block.number, transaction: transaction, data: "0x010101", - first_topic: "0x01" + first_topic: topic(@first_topic_hex_string_1) ) insert(:log, @@ -143,10 +154,13 @@ defmodule BlockScoutWeb.API.RPC.EthControllerTest do block_number: block.number, transaction: transaction, data: "0x020202", - first_topic: "0x00" + first_topic: topic(@first_topic_hex_string_2) ) - params = params(api_params, [%{"address" => to_string(address.hash), "topics" => [["0x01", "0x00"]]}]) + params = + params(api_params, [ + %{"address" => to_string(address.hash), "topics" => [[@first_topic_hex_string_1, @first_topic_hex_string_2]]} + ]) assert response = conn @@ -171,10 +185,11 @@ defmodule BlockScoutWeb.API.RPC.EthControllerTest do block_number: block.number, address: contract_address, transaction: transaction, - first_topic: "0x01" + first_topic: topic(@first_topic_hex_string_1) ) - params = params(api_params, [%{"address" => to_string(contract_address), "topics" => [["0x01"]]}]) + params = + params(api_params, [%{"address" => to_string(contract_address), "topics" => [[@first_topic_hex_string_1]]}]) assert response = conn @@ -192,7 +207,11 @@ defmodule BlockScoutWeb.API.RPC.EthControllerTest do new_params = params(api_params, [ - %{"paging_options" => next_page_params, "address" => to_string(contract_address), "topics" => [["0x01"]]} + %{ + "paging_options" => next_page_params, + "address" => to_string(contract_address), + "topics" => [[@first_topic_hex_string_1]] + } ]) assert new_response = @@ -227,15 +246,24 @@ defmodule BlockScoutWeb.API.RPC.EthControllerTest do address: address, transaction: transaction, data: "0x010101", - first_topic: "0x01", - second_topic: "0x02", + first_topic: topic(@first_topic_hex_string_1), + second_topic: topic(@second_topic_hex_string_1), block: block, block_number: block.number ) - insert(:log, block: block, address: address, transaction: transaction, data: "0x020202", first_topic: "0x01") + insert(:log, + block: block, + address: address, + transaction: transaction, + data: "0x020202", + first_topic: topic(@first_topic_hex_string_1) + ) - params = params(api_params, [%{"address" => to_string(address.hash), "topics" => ["0x01", "0x02"]}]) + params = + params(api_params, [ + %{"address" => to_string(address.hash), "topics" => [@first_topic_hex_string_1, @second_topic_hex_string_1]} + ]) assert response = conn @@ -257,8 +285,8 @@ defmodule BlockScoutWeb.API.RPC.EthControllerTest do address: address, transaction: transaction, data: "0x010101", - first_topic: "0x01", - second_topic: "0x02", + first_topic: topic(@first_topic_hex_string_1), + second_topic: topic(@second_topic_hex_string_1), block: block, block_number: block.number ) @@ -267,13 +295,19 @@ defmodule BlockScoutWeb.API.RPC.EthControllerTest do address: address, transaction: transaction, data: "0x020202", - first_topic: "0x01", - second_topic: "0x03", + first_topic: topic(@first_topic_hex_string_1), + second_topic: topic(@second_topic_hex_string_2), block: block, block_number: block.number ) - params = params(api_params, [%{"address" => to_string(address.hash), "topics" => ["0x01", ["0x02", "0x03"]]}]) + params = + params(api_params, [ + %{ + "address" => to_string(address.hash), + "topics" => [@first_topic_hex_string_1, [@second_topic_hex_string_1, @second_topic_hex_string_2]] + } + ]) assert response = conn diff --git a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/logs_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/logs_controller_test.exs index 66fc27dc71..2691f1e7e8 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/logs_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/logs_controller_test.exs @@ -4,6 +4,22 @@ defmodule BlockScoutWeb.API.RPC.LogsControllerTest do alias BlockScoutWeb.API.RPC.LogsController alias Explorer.Chain.{Log, Transaction} + @first_topic_hex_string_1 "0x7fcf532c15f0a6db0bd6d0e038bea71d30d808c7d98cb3bf7268a95bf5081b65" + @first_topic_hex_string_2 "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef" + + @second_topic_hex_string_1 "0x00000000000000000000000098a9dc37d3650b5b30d6c12789b3881ee0b70c16" + @second_topic_hex_string_2 "0x000000000000000000000000e2680fd7cdbb04e9087a647ad4d023ef6c8fb4e2" + + @third_topic_hex_string_1 "0x0000000000000000000000005079fc00f00f30000e0c8c083801cfde000008b6" + + @fourth_topic_hex_string_1 "0x8c9b7729443a4444242342b2ca385a239a5c1d76a88473e1cd2ab0c70dd1b9c7" + @fourth_topic_hex_string_2 "0x232b688786cc0d24a11e07563c1bfa129537cec9385dc5b1fb8f86462977239b" + + defp topic(topic_hex_string) do + {:ok, topic} = Explorer.Chain.Hash.Full.cast(topic_hex_string) + topic + end + describe "getLogs" do test "without fromBlock, toBlock, address, and topic{x}", %{conn: conn} do params = %{ @@ -434,13 +450,13 @@ defmodule BlockScoutWeb.API.RPC.LogsControllerTest do log1_details = [ address: contract_address, transaction: transaction, - first_topic: "some topic" + first_topic: topic(@first_topic_hex_string_1) ] log2_details = [ address: contract_address, transaction: transaction, - first_topic: "some other topic" + first_topic: topic(@first_topic_hex_string_2) ] log1 = insert(:log, log1_details) @@ -492,15 +508,15 @@ defmodule BlockScoutWeb.API.RPC.LogsControllerTest do log1_details = [ address: contract_address, transaction: transaction, - first_topic: "some topic", - second_topic: "some second topic" + first_topic: topic(@first_topic_hex_string_1), + second_topic: topic(@second_topic_hex_string_1) ] log2_details = [ address: contract_address, transaction: transaction, - first_topic: "some other topic", - second_topic: "some other second topic" + first_topic: topic(@first_topic_hex_string_2), + second_topic: topic(@second_topic_hex_string_2) ] log1 = insert(:log, log1_details) @@ -541,15 +557,15 @@ defmodule BlockScoutWeb.API.RPC.LogsControllerTest do log1_details = [ address: contract_address, transaction: transaction, - first_topic: "some topic", - second_topic: "some second topic" + first_topic: topic(@first_topic_hex_string_1), + second_topic: topic(@second_topic_hex_string_1) ] log2_details = [ address: contract_address, transaction: transaction, - first_topic: "some other topic", - second_topic: "some other second topic" + first_topic: topic(@first_topic_hex_string_2), + second_topic: topic(@second_topic_hex_string_2) ] log1 = insert(:log, log1_details) @@ -589,19 +605,19 @@ defmodule BlockScoutWeb.API.RPC.LogsControllerTest do log1_details = [ address: contract_address, transaction: transaction, - first_topic: "some topic", - second_topic: "some second topic", - third_topic: "some third topic", - fourth_topic: "some fourth topic" + first_topic: topic(@first_topic_hex_string_1), + second_topic: topic(@second_topic_hex_string_1), + third_topic: topic(@third_topic_hex_string_1), + fourth_topic: topic(@fourth_topic_hex_string_1) ] log2_details = [ address: contract_address, transaction: transaction, - first_topic: "some topic", - second_topic: "some second topic", - third_topic: "some third topic", - fourth_topic: "some other fourth topic" + first_topic: topic(@first_topic_hex_string_1), + second_topic: topic(@second_topic_hex_string_1), + third_topic: topic(@third_topic_hex_string_1), + fourth_topic: topic(@fourth_topic_hex_string_2) ] log1 = insert(:log, log1_details) @@ -791,7 +807,12 @@ defmodule BlockScoutWeb.API.RPC.LogsControllerTest do third_topic: third_topic, fourth_topic: fourth_topic }) do - [first_topic, second_topic, third_topic, fourth_topic] + [ + first_topic && Explorer.Chain.Hash.to_string(first_topic), + second_topic && Explorer.Chain.Hash.to_string(second_topic), + third_topic && Explorer.Chain.Hash.to_string(third_topic), + fourth_topic && Explorer.Chain.Hash.to_string(fourth_topic) + ] end defp integer_to_hex(integer), do: "0x" <> String.downcase(Integer.to_string(integer, 16)) diff --git a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/transaction_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/transaction_controller_test.exs index 4a828099ce..8e30a3e680 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/transaction_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/transaction_controller_test.exs @@ -5,8 +5,16 @@ defmodule BlockScoutWeb.API.RPC.TransactionControllerTest do @moduletag capture_log: true + @first_topic_hex_string_1 "0x7fcf532c15f0a6db0bd6d0e038bea71d30d808c7d98cb3bf7268a95bf5081b65" + @second_topic_hex_string_1 "0x00000000000000000000000098a9dc37d3650b5b30d6c12789b3881ee0b70c16" + setup :verify_on_exit! + defp topic(topic_hex_string) do + {:ok, topic} = Explorer.Chain.Hash.Full.cast(topic_hex_string) + topic + end + describe "gettxreceiptstatus" do test "with missing txhash", %{conn: conn} do params = %{ @@ -414,8 +422,8 @@ defmodule BlockScoutWeb.API.RPC.TransactionControllerTest do insert(:log, address: address, transaction: transaction, - first_topic: "first topic", - second_topic: "second topic", + first_topic: topic(@first_topic_hex_string_1), + second_topic: topic(@second_topic_hex_string_1), block: block, block_number: block.number ) @@ -491,8 +499,8 @@ defmodule BlockScoutWeb.API.RPC.TransactionControllerTest do insert(:log, address: address, transaction: transaction, - first_topic: "first topic", - second_topic: "second topic", + first_topic: topic(@first_topic_hex_string_1), + second_topic: topic(@second_topic_hex_string_1), block: block, block_number: block.number ) @@ -520,7 +528,7 @@ defmodule BlockScoutWeb.API.RPC.TransactionControllerTest do %{ "address" => "#{address.hash}", "data" => "#{log.data}", - "topics" => ["first topic", "second topic", nil, nil], + "topics" => [@first_topic_hex_string_1, @second_topic_hex_string_1, nil, nil], "index" => "#{log.index}" } ], diff --git a/apps/block_scout_web/test/block_scout_web/controllers/api/v2/address_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/api/v2/address_controller_test.exs index d3749c1628..34829785f5 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/api/v2/address_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/api/v2/address_controller_test.exs @@ -26,12 +26,18 @@ defmodule BlockScoutWeb.API.V2.AddressControllerTest do import Explorer.Chain, only: [hash_to_lower_case_string: 1] import Mox + @first_topic_hex_string_1 "0x7fcf532c15f0a6db0bd6d0e038bea71d30d808c7d98cb3bf7268a95bf5081b65" @instances_amount_in_collection 9 setup :set_mox_global setup :verify_on_exit! + defp topic(topic_hex_string) do + {:ok, topic} = Explorer.Chain.Hash.Full.cast(topic_hex_string) + topic + end + describe "/addresses/{address_hash}" do test "get 404 on non existing address", %{conn: conn} do address = build(:address) @@ -1761,10 +1767,10 @@ defmodule BlockScoutWeb.API.V2.AddressControllerTest do block: tx.block, block_number: tx.block_number, address: address, - first_topic: "0x123456789123456789" + first_topic: topic(@first_topic_hex_string_1) ) - request = get(conn, "/api/v2/addresses/#{address.hash}/logs?topic=0x123456789123456789") + request = get(conn, "/api/v2/addresses/#{address.hash}/logs?topic=#{@first_topic_hex_string_1}") assert response = json_response(request, 200) assert Enum.count(response["items"]) == 1 assert response["next_page_params"] == nil diff --git a/apps/block_scout_web/test/block_scout_web/controllers/api/v2/transaction_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/api/v2/transaction_controller_test.exs index ede7f16822..9fdad1782d 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/api/v2/transaction_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/api/v2/transaction_controller_test.exs @@ -8,6 +8,13 @@ defmodule BlockScoutWeb.API.V2.TransactionControllerTest do alias Explorer.Chain.{Address, InternalTransaction, Log, Token, TokenTransfer, Transaction} alias Explorer.Repo + @first_topic_hex_string_1 "0x99e7b0ba56da2819c37c047f0511fd2bf6c9b4e27b4a979a19d6da0f74be8155" + + defp topic(topic_hex_string) do + {:ok, topic} = Explorer.Chain.Hash.Full.cast(topic_hex_string) + topic + end + setup do Supervisor.terminate_child(Explorer.Supervisor, Explorer.Chain.Cache.TransactionsApiV2.child_id()) Supervisor.restart_child(Explorer.Supervisor, Explorer.Chain.Cache.TransactionsApiV2.child_id()) @@ -976,7 +983,7 @@ defmodule BlockScoutWeb.API.V2.TransactionControllerTest do index: 1, block: tx.block, block_number: tx.block_number, - first_topic: "0x99e7b0ba56da2819c37c047f0511fd2bf6c9b4e27b4a979a19d6da0f74be8155", + first_topic: topic(@first_topic_hex_string_1), data: "0x000000000000000000000000dc2b93f3291030f3f7a6d9363ac37757f7ad5c4300000000000000000000000000000000000000000000000000002824369a100000000000000000000000000046b555cb3962bf9533c437cbd04a2f702dfdb999000000000000000000000000000000000000000000000000000014121b4d0800000000000000000000000000faf7a981360c2fab3a5ab7b3d6d8d0cf97a91eb9000000000000000000000000000000000000000000000000000014121b4d0800" ) @@ -1039,7 +1046,7 @@ defmodule BlockScoutWeb.API.V2.TransactionControllerTest do index: 1, block: tx.block, block_number: tx.block_number, - first_topic: "0x99e7b0ba56da2819c37c047f0511fd2bf6c9b4e27b4a979a19d6da0f74be8155", + first_topic: topic(@first_topic_hex_string_1), data: "0x000000000000000000000000dc2b93f3291030f3f7a6d9363ac37757f7ad5c4300000000000000000000000000000000000000000000000000002824369a100000000000000000000000000046b555cb3962bf9533c437cbd04a2f702dfdb999000000000000000000000000000000000000000000000000000014121b4d0800000000000000000000000000faf7a981360c2fab3a5ab7b3d6d8d0cf97a91eb9000000000000000000000000000000000000000000000000000014121b4d0800" ) diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index 19983d0045..f6a4e3615f 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -3590,32 +3590,6 @@ defmodule Explorer.Chain do |> Repo.stream_reduce(initial, reducer) end - @doc """ - Returns a list of block numbers token transfer `t:Log.t/0`s that don't have an - associated `t:TokenTransfer.t/0` record. - """ - def uncataloged_token_transfer_block_numbers do - query = - from(l in Log, - as: :log, - where: - l.first_topic == unquote(TokenTransfer.constant()) or - l.first_topic == unquote(TokenTransfer.erc1155_single_transfer_signature()) or - l.first_topic == unquote(TokenTransfer.erc1155_batch_transfer_signature()), - where: - not exists( - from(tf in TokenTransfer, - where: tf.transaction_hash == parent_as(:log).transaction_hash, - where: tf.log_index == parent_as(:log).index - ) - ), - select: l.block_number, - distinct: l.block_number - ) - - Repo.stream_reduce(query, [], &[&1 | &2]) - end - def decode_contract_address_hash_response(resp) do case resp do "0x000000000000000000000000" <> address -> diff --git a/apps/explorer/lib/explorer/chain/log.ex b/apps/explorer/lib/explorer/chain/log.ex index 2eb2c2ce92..e1c87a8a1e 100644 --- a/apps/explorer/lib/explorer/chain/log.ex +++ b/apps/explorer/lib/explorer/chain/log.ex @@ -35,10 +35,10 @@ defmodule Explorer.Chain.Log do block_hash: Hash.Full.t(), block_number: non_neg_integer() | nil, data: Data.t(), - first_topic: String.t(), - second_topic: String.t(), - third_topic: String.t(), - fourth_topic: String.t(), + first_topic: Hash.Full.t(), + second_topic: Hash.Full.t(), + third_topic: Hash.Full.t(), + fourth_topic: Hash.Full.t(), transaction: %Ecto.Association.NotLoaded{} | Transaction.t(), transaction_hash: Hash.Full.t(), index: non_neg_integer(), @@ -48,10 +48,10 @@ defmodule Explorer.Chain.Log do @primary_key false schema "logs" do field(:data, Data) - field(:first_topic, :string) - field(:second_topic, :string) - field(:third_topic, :string) - field(:fourth_topic, :string) + field(:first_topic, Hash.Full) + field(:second_topic, Hash.Full) + field(:third_topic, Hash.Full) + field(:fourth_topic, Hash.Full) field(:index, :integer, primary_key: true) field(:type, :string) field(:block_number, :integer) @@ -190,9 +190,10 @@ defmodule Explorer.Chain.Log do end defp find_method_candidates(log, transaction, options, events_acc, skip_sig_provider?) do - with "0x" <> hex_part <- log.first_topic, - {number, ""} <- Integer.parse(hex_part, 16) do - <> = :binary.encode_unsigned(number) + if is_nil(log.first_topic) do + {{:error, :could_not_decode}, events_acc} + else + <> = log.first_topic.bytes if Map.has_key?(events_acc, method_id) do {events_acc[method_id], events_acc} @@ -200,8 +201,6 @@ defmodule Explorer.Chain.Log do result = find_method_candidates_from_db(method_id, log, transaction, options, skip_sig_provider?) {result, Map.put(events_acc, method_id, result)} end - else - _ -> {{:error, :could_not_decode}, events_acc} end end @@ -243,10 +242,10 @@ defmodule Explorer.Chain.Log do abi |> ABI.parse_specification(include_events?: true) |> Event.find_and_decode( - decode16!(log.first_topic), - decode16!(log.second_topic), - decode16!(log.third_topic), - decode16!(log.fourth_topic), + log.first_topic && log.first_topic.bytes, + log.second_topic && log.second_topic.bytes, + log.third_topic && log.third_topic.bytes, + log.fourth_topic && log.fourth_topic.bytes, log.data.bytes ) do {:ok, selector, mapping} diff --git a/apps/explorer/lib/explorer/chain/token_transfer.ex b/apps/explorer/lib/explorer/chain/token_transfer.ex index 8b9b878914..c53414e209 100644 --- a/apps/explorer/lib/explorer/chain/token_transfer.ex +++ b/apps/explorer/lib/explorer/chain/token_transfer.ex @@ -28,7 +28,7 @@ defmodule Explorer.Chain.TokenTransfer do import Ecto.Query, only: [from: 2, limit: 2, where: 3, join: 5, order_by: 3, preload: 3] alias Explorer.Chain - alias Explorer.Chain.{Address, Block, DenormalizationHelper, Hash, TokenTransfer, Transaction} + alias Explorer.Chain.{Address, Block, DenormalizationHelper, Hash, Log, TokenTransfer, Transaction} alias Explorer.Chain.Token.Instance alias Explorer.{PagingOptions, Repo} @@ -370,4 +370,31 @@ defmodule Explorer.Chain.TokenTransfer do where: block.consensus == true ) end + + @doc """ + Returns a list of block numbers token transfer `t:Log.t/0`s that don't have an + associated `t:TokenTransfer.t/0` record. + """ + @spec uncataloged_token_transfer_block_numbers :: {:ok, [non_neg_integer()]} + def uncataloged_token_transfer_block_numbers do + query = + from(l in Log, + as: :log, + where: + l.first_topic == ^@constant or + l.first_topic == ^@erc1155_single_transfer_signature or + l.first_topic == ^@erc1155_batch_transfer_signature, + where: + not exists( + from(tf in TokenTransfer, + where: tf.transaction_hash == parent_as(:log).transaction_hash, + where: tf.log_index == parent_as(:log).index + ) + ), + select: l.block_number, + distinct: l.block_number + ) + + Repo.stream_reduce(query, [], &[&1 | &2]) + end end diff --git a/apps/explorer/priv/repo/migrations/20231213152332_alter_log_topic_columns_type.exs b/apps/explorer/priv/repo/migrations/20231213152332_alter_log_topic_columns_type.exs new file mode 100644 index 0000000000..649c95d0db --- /dev/null +++ b/apps/explorer/priv/repo/migrations/20231213152332_alter_log_topic_columns_type.exs @@ -0,0 +1,18 @@ +defmodule Explorer.Repo.Migrations.AlterLogTopicColumnsType do + use Ecto.Migration + + def change do + execute(""" + ALTER TABLE logs + ALTER COLUMN first_topic TYPE bytea + USING CAST(REPLACE(first_topic, '0x', '\\x') as bytea), + ALTER COLUMN second_topic TYPE bytea + USING CAST(REPLACE(second_topic, '0x', '\\x') as bytea), + ALTER COLUMN third_topic TYPE bytea + USING CAST(REPLACE(third_topic, '0x', '\\x') as bytea), + ALTER COLUMN fourth_topic TYPE bytea + USING CAST(REPLACE(fourth_topic, '0x', '\\x') as bytea) + ; + """) + end +end diff --git a/apps/explorer/test/explorer/chain/csv_export/address_log_csv_exporter_test.exs b/apps/explorer/test/explorer/chain/csv_export/address_log_csv_exporter_test.exs index bff1ca818d..e70b2eb9cb 100644 --- a/apps/explorer/test/explorer/chain/csv_export/address_log_csv_exporter_test.exs +++ b/apps/explorer/test/explorer/chain/csv_export/address_log_csv_exporter_test.exs @@ -4,6 +4,16 @@ defmodule Explorer.Chain.AddressLogCsvExporterTest do alias Explorer.Chain.Address alias Explorer.Chain.CSVExport.AddressLogCsvExporter + @first_topic_hex_string_1 "0x7fcf532c15f0a6db0bd6d0e038bea71d30d808c7d98cb3bf7268a95bf5081b65" + @second_topic_hex_string_1 "0x00000000000000000000000098a9dc37d3650b5b30d6c12789b3881ee0b70c16" + @third_topic_hex_string_1 "0x0000000000000000000000005079fc00f00f30000e0c8c083801cfde000008b6" + @fourth_topic_hex_string_1 "0x8c9b7729443a4444242342b2ca385a239a5c1d76a88473e1cd2ab0c70dd1b9c7" + + defp topic(topic_hex_string) do + {:ok, topic} = Explorer.Chain.Hash.Full.cast(topic_hex_string) + topic + end + describe "export/3" do test "exports address logs to csv" do address = insert(:address) @@ -21,10 +31,10 @@ defmodule Explorer.Chain.AddressLogCsvExporterTest do block: transaction.block, block_number: transaction.block_number, data: "0x12", - first_topic: "0x13", - second_topic: "0x14", - third_topic: "0x15", - fourth_topic: "0x16" + first_topic: topic(@first_topic_hex_string_1), + second_topic: topic(@second_topic_hex_string_1), + third_topic: topic(@third_topic_hex_string_1), + fourth_topic: topic(@fourth_topic_hex_string_1) ) from_period = Timex.format!(Timex.shift(Timex.now(), minutes: -1), "%Y-%m-%d", :strftime) diff --git a/apps/explorer/test/explorer/chain/import_test.exs b/apps/explorer/test/explorer/chain/import_test.exs index 322ca7a7ff..a8d62bfcad 100644 --- a/apps/explorer/test/explorer/chain/import_test.exs +++ b/apps/explorer/test/explorer/chain/import_test.exs @@ -22,9 +22,16 @@ defmodule Explorer.Chain.ImportTest do @moduletag :capturelog + @first_topic_hex_string "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef" + @second_topic_hex_string "0x000000000000000000000000e8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca" + @third_topic_hex_string "0x000000000000000000000000515c09c5bba1ed566b02a5b0599ec5d5d0aee73d" + doctest Import describe "all/1" do + {:ok, first_topic} = Explorer.Chain.Hash.Full.cast(@first_topic_hex_string) + {:ok, second_topic} = Explorer.Chain.Hash.Full.cast(@second_topic_hex_string) + {:ok, third_topic} = Explorer.Chain.Hash.Full.cast(@third_topic_hex_string) # set :timeout options to cover lines that use the timeout override when available @import_data %{ blocks: %{ @@ -91,9 +98,9 @@ defmodule Explorer.Chain.ImportTest do block_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd", address_hash: "0x8bf38d4764929064f2d4d3a56520a76ab3df415b", data: "0x0000000000000000000000000000000000000000000000000de0b6b3a7640000", - first_topic: "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", - second_topic: "0x000000000000000000000000e8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca", - third_topic: "0x000000000000000000000000515c09c5bba1ed566b02a5b0599ec5d5d0aee73d", + first_topic: first_topic, + second_topic: second_topic, + third_topic: third_topic, fourth_topic: nil, index: 0, transaction_hash: "0x53bd884872de3e488692881baeec262e7b95234d3965248c39fe992fffd433e5", @@ -165,6 +172,9 @@ defmodule Explorer.Chain.ImportTest do } test "with valid data" do + {:ok, first_topic} = Explorer.Chain.Hash.Full.cast(@first_topic_hex_string) + {:ok, second_topic} = Explorer.Chain.Hash.Full.cast(@second_topic_hex_string) + {:ok, third_topic} = Explorer.Chain.Hash.Full.cast(@third_topic_hex_string) difficulty = Decimal.new(340_282_366_920_938_463_463_374_607_431_768_211_454) total_difficulty = Decimal.new(12_590_447_576_074_723_148_144_860_474_975_121_280_509) token_transfer_amount = Decimal.new(1_000_000_000_000_000_000) @@ -276,9 +286,9 @@ defmodule Explorer.Chain.ImportTest do 167, 100, 0, 0>> }, index: 0, - first_topic: "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", - second_topic: "0x000000000000000000000000e8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca", - third_topic: "0x000000000000000000000000515c09c5bba1ed566b02a5b0599ec5d5d0aee73d", + first_topic: ^first_topic, + second_topic: ^second_topic, + third_topic: ^third_topic, fourth_topic: nil, transaction_hash: %Hash{ byte_count: 32, diff --git a/apps/explorer/test/explorer/chain/log_test.exs b/apps/explorer/test/explorer/chain/log_test.exs index 6842a6b979..87881776de 100644 --- a/apps/explorer/test/explorer/chain/log_test.exs +++ b/apps/explorer/test/explorer/chain/log_test.exs @@ -7,6 +7,13 @@ defmodule Explorer.Chain.LogTest do alias Explorer.Chain.{Log, SmartContract} alias Explorer.Repo + @first_topic_hex_string_1 "0x7fcf532c15f0a6db0bd6d0e038bea71d30d808c7d98cb3bf7268a95bf5081b65" + + defp topic(topic_hex_string) do + {:ok, topic} = Explorer.Chain.Hash.Full.cast(topic_hex_string) + topic + end + setup :set_mox_from_context doctest Log @@ -36,18 +43,21 @@ defmodule Explorer.Chain.LogTest do params_for( :log, address_hash: build(:address).hash, - first_topic: "ham", + first_topic: @first_topic_hex_string_1, transaction_hash: build(:transaction).hash, block_hash: build(:block).hash ) - assert %Changeset{changes: %{first_topic: "ham"}, valid?: true} = Log.changeset(%Log{}, params) + result = Log.changeset(%Log{}, params) + + assert result.valid? == true + assert result.changes.first_topic == topic(@first_topic_hex_string_1) end test "assigns optional attributes" do - params = Map.put(params_for(:log), :first_topic, "ham") + params = Map.put(params_for(:log), :first_topic, topic(@first_topic_hex_string_1)) changeset = Log.changeset(%Log{}, params) - assert changeset.changes.first_topic === "ham" + assert changeset.changes.first_topic === topic(@first_topic_hex_string_1) end end @@ -99,9 +109,9 @@ defmodule Explorer.Chain.LogTest do insert(:log, address: to_address, transaction: transaction, - first_topic: topic1, - second_topic: topic2, - third_topic: topic3, + first_topic: topic(topic1), + second_topic: topic(topic2), + third_topic: topic(topic3), fourth_topic: nil, data: data ) @@ -153,9 +163,9 @@ defmodule Explorer.Chain.LogTest do log = insert(:log, transaction: transaction, - first_topic: topic1, - second_topic: topic2, - third_topic: topic3, + first_topic: topic(topic1), + second_topic: topic(topic2), + third_topic: topic(topic3), fourth_topic: nil, data: data ) diff --git a/apps/explorer/test/explorer/chain/smart_contract_test.exs b/apps/explorer/test/explorer/chain/smart_contract_test.exs index 8bd0b1e581..f7c45251b0 100644 --- a/apps/explorer/test/explorer/chain/smart_contract_test.exs +++ b/apps/explorer/test/explorer/chain/smart_contract_test.exs @@ -2,9 +2,8 @@ defmodule Explorer.Chain.SmartContractTest do use Explorer.DataCase, async: false import Mox - alias Explorer.{Chain, PagingOptions} + alias Explorer.Chain alias Explorer.Chain.{Address, SmartContract} - alias Explorer.Chain.Hash alias Explorer.Chain.SmartContract.Proxy doctest Explorer.Chain.SmartContract diff --git a/apps/explorer/test/explorer/chain/token_transfer_test.exs b/apps/explorer/test/explorer/chain/token_transfer_test.exs index 29165457b7..3139f318cb 100644 --- a/apps/explorer/test/explorer/chain/token_transfer_test.exs +++ b/apps/explorer/test/explorer/chain/token_transfer_test.exs @@ -325,4 +325,28 @@ defmodule Explorer.Chain.TokenTransferTest do assert Enum.member?(page_two, transaction_one_bytes) == true end end + + describe "uncataloged_token_transfer_block_numbers/0" do + test "returns a list of block numbers" do + block = insert(:block) + address = insert(:address) + + log = + insert(:token_transfer_log, + transaction: + insert(:transaction, + block_number: block.number, + block_hash: block.hash, + cumulative_gas_used: 0, + gas_used: 0, + index: 0 + ), + block: block, + address_hash: address.hash + ) + + block_number = log.block_number + assert {:ok, [^block_number]} = TokenTransfer.uncataloged_token_transfer_block_numbers() + end + end end diff --git a/apps/explorer/test/explorer/chain_test.exs b/apps/explorer/test/explorer/chain_test.exs index 2bbbaebcd2..083d50d0f6 100644 --- a/apps/explorer/test/explorer/chain_test.exs +++ b/apps/explorer/test/explorer/chain_test.exs @@ -23,7 +23,6 @@ defmodule Explorer.ChainTest do Token, TokenTransfer, Transaction, - SmartContract, Wei } @@ -38,6 +37,10 @@ defmodule Explorer.ChainTest do alias Explorer.Counters.AddressesWithBalanceCounter alias Explorer.Counters.AddressesCounter + @first_topic_hex_string "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef" + @second_topic_hex_string "0x000000000000000000000000e8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca" + @third_topic_hex_string "0x000000000000000000000000515c09c5bba1ed566b02a5b0599ec5d5d0aee73d" + doctest Explorer.Chain setup :set_mox_global @@ -316,6 +319,9 @@ defmodule Explorer.ChainTest do block_number: transaction1.block_number ) + first_topic_hex_string = "0x7fcf532c15f0a6db0bd6d0e038bea71d30d808c7d98cb3bf7268a95bf5081b65" + {:ok, first_topic} = Explorer.Chain.Hash.Full.cast(first_topic_hex_string) + transaction2 = :transaction |> insert(from_address: address) @@ -326,11 +332,11 @@ defmodule Explorer.ChainTest do transaction: transaction2, index: 2, address: address, - first_topic: "test", + first_topic: first_topic, block_number: transaction2.block_number ) - [found_log] = Chain.address_to_logs(address_hash, false, topic: "test") + [found_log] = Chain.address_to_logs(address_hash, false, topic: first_topic_hex_string) assert found_log.transaction.hash == transaction2.hash end @@ -343,13 +349,16 @@ defmodule Explorer.ChainTest do |> insert(to_address: address) |> with_block() + fourth_topic_hex_string = "0x927abf391899d10d331079a63caffa905efa7075a44a7bbd52b190db4c4308fb" + {:ok, fourth_topic} = Explorer.Chain.Hash.Full.cast(fourth_topic_hex_string) + insert(:log, block: transaction1.block, block_number: transaction1.block_number, transaction: transaction1, index: 1, address: address, - fourth_topic: "test" + fourth_topic: fourth_topic ) transaction2 = @@ -365,7 +374,7 @@ defmodule Explorer.ChainTest do address: address ) - [found_log] = Chain.address_to_logs(address_hash, false, topic: "test") + [found_log] = Chain.address_to_logs(address_hash, false, topic: fourth_topic_hex_string) assert found_log.transaction.hash == transaction1.hash end @@ -1238,6 +1247,10 @@ defmodule Explorer.ChainTest do # Full tests in `test/explorer/import_test.exs` describe "import/1" do + {:ok, first_topic} = Explorer.Chain.Hash.Full.cast(@first_topic_hex_string) + {:ok, second_topic} = Explorer.Chain.Hash.Full.cast(@second_topic_hex_string) + {:ok, third_topic} = Explorer.Chain.Hash.Full.cast(@third_topic_hex_string) + @import_data %{ blocks: %{ params: [ @@ -1293,9 +1306,9 @@ defmodule Explorer.ChainTest do block_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd", address_hash: "0x8bf38d4764929064f2d4d3a56520a76ab3df415b", data: "0x0000000000000000000000000000000000000000000000000de0b6b3a7640000", - first_topic: "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", - second_topic: "0x000000000000000000000000e8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca", - third_topic: "0x000000000000000000000000515c09c5bba1ed566b02a5b0599ec5d5d0aee73d", + first_topic: first_topic, + second_topic: second_topic, + third_topic: third_topic, fourth_topic: nil, index: 0, transaction_hash: "0x53bd884872de3e488692881baeec262e7b95234d3965248c39fe992fffd433e5", @@ -1362,6 +1375,9 @@ defmodule Explorer.ChainTest do } test "with valid data" do + {:ok, first_topic} = Explorer.Chain.Hash.Full.cast(@first_topic_hex_string) + {:ok, second_topic} = Explorer.Chain.Hash.Full.cast(@second_topic_hex_string) + {:ok, third_topic} = Explorer.Chain.Hash.Full.cast(@third_topic_hex_string) difficulty = Decimal.new(340_282_366_920_938_463_463_374_607_431_768_211_454) total_difficulty = Decimal.new(12_590_447_576_074_723_148_144_860_474_975_121_280_509) token_transfer_amount = Decimal.new(1_000_000_000_000_000_000) @@ -1464,9 +1480,9 @@ defmodule Explorer.ChainTest do 167, 100, 0, 0>> }, index: 0, - first_topic: "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", - second_topic: "0x000000000000000000000000e8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca", - third_topic: "0x000000000000000000000000515c09c5bba1ed566b02a5b0599ec5d5d0aee73d", + first_topic: ^first_topic, + second_topic: ^second_topic, + third_topic: ^third_topic, fourth_topic: nil, transaction_hash: %Hash{ byte_count: 32, @@ -4367,30 +4383,6 @@ defmodule Explorer.ChainTest do end end - describe "uncataloged_token_transfer_block_numbers/0" do - test "returns a list of block numbers" do - block = insert(:block) - address = insert(:address) - - log = - insert(:token_transfer_log, - transaction: - insert(:transaction, - block_number: block.number, - block_hash: block.hash, - cumulative_gas_used: 0, - gas_used: 0, - index: 0 - ), - block: block, - address_hash: address.hash - ) - - block_number = log.block_number - assert {:ok, [^block_number]} = Chain.uncataloged_token_transfer_block_numbers() - end - end - describe "address_to_balances_by_day/1" do test "return a list of balances by day" do address = insert(:address) diff --git a/apps/explorer/test/explorer/etherscan/logs_test.exs b/apps/explorer/test/explorer/etherscan/logs_test.exs index 5da949541c..b5953407f8 100644 --- a/apps/explorer/test/explorer/etherscan/logs_test.exs +++ b/apps/explorer/test/explorer/etherscan/logs_test.exs @@ -6,6 +6,25 @@ defmodule Explorer.Etherscan.LogsTest do alias Explorer.Etherscan.Logs alias Explorer.Chain.Transaction + @first_topic_hex_string_1 "0x7fcf532c15f0a6db0bd6d0e038bea71d30d808c7d98cb3bf7268a95bf5081b65" + @first_topic_hex_string_2 "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef" + @first_topic_hex_string_3 "0xe1fffcc4923d04b559f4d29a8bfc6cda04eb5b0d3c460751c2402c5c5cc9109c" + + @second_topic_hex_string_1 "0x00000000000000000000000098a9dc37d3650b5b30d6c12789b3881ee0b70c16" + @second_topic_hex_string_2 "0x000000000000000000000000e2680fd7cdbb04e9087a647ad4d023ef6c8fb4e2" + @second_topic_hex_string_3 "0x0000000000000000000000005777d92f208679db4b9778590fa3cab3ac9e2168" + + @third_topic_hex_string_1 "0x0000000000000000000000005079fc00f00f30000e0c8c083801cfde000008b6" + @third_topic_hex_string_2 "0x000000000000000000000000e2680fd7cdbb04e9087a647ad4d023ef6c8fb4e2" + @third_topic_hex_string_3 "0x0000000000000000000000000f6d9bd6fc315bbf95b5c44f4eba2b2762f8c372" + + @fourth_topic_hex_string_1 "0x8c9b7729443a4444242342b2ca385a239a5c1d76a88473e1cd2ab0c70dd1b9c7" + + defp topic(topic_hex_string) do + {:ok, topic} = Explorer.Chain.Hash.Full.cast(topic_hex_string) + topic + end + describe "list_logs/1" do test "with empty db" do contract_address = build(:contract_address) @@ -211,13 +230,13 @@ defmodule Explorer.Etherscan.LogsTest do log1_details = [ address: contract_address, transaction: transaction, - first_topic: "some topic" + first_topic: topic(@first_topic_hex_string_1) ] log2_details = [ address: contract_address, transaction: transaction, - first_topic: "some other topic" + first_topic: topic(@first_topic_hex_string_2) ] log1 = insert(:log, log1_details) @@ -247,15 +266,15 @@ defmodule Explorer.Etherscan.LogsTest do log1_details = [ address: contract_address, transaction: transaction, - first_topic: "some first topic", - second_topic: "some second topic" + first_topic: topic(@first_topic_hex_string_1), + second_topic: topic(@second_topic_hex_string_1) ] log2_details = [ address: contract_address, transaction: transaction, - first_topic: "some first topic", - second_topic: "some OTHER second topic" + first_topic: topic(@first_topic_hex_string_1), + second_topic: topic(@second_topic_hex_string_2) ] _log1 = insert(:log, log1_details) @@ -288,15 +307,15 @@ defmodule Explorer.Etherscan.LogsTest do log1_details = [ address: contract_address, transaction: transaction, - first_topic: "some first topic", - second_topic: "some second topic" + first_topic: topic(@first_topic_hex_string_1), + second_topic: topic(@second_topic_hex_string_1) ] log2_details = [ address: contract_address, transaction: transaction, - first_topic: "some OTHER first topic", - second_topic: "some OTHER second topic" + first_topic: topic(@first_topic_hex_string_2), + second_topic: topic(@second_topic_hex_string_2) ] log1 = insert(:log, log1_details) @@ -327,14 +346,14 @@ defmodule Explorer.Etherscan.LogsTest do log1_details = [ address: contract_address, transaction: transaction, - first_topic: "some first topic", + first_topic: topic(@first_topic_hex_string_1), block_number: block.number ] log2_details = [ address: contract_address, transaction: transaction, - first_topic: "some OTHER first topic", + first_topic: topic(@first_topic_hex_string_2), block_number: block.number ] @@ -366,16 +385,16 @@ defmodule Explorer.Etherscan.LogsTest do log1_details = [ address: contract_address, transaction: transaction, - first_topic: "some first topic", - second_topic: "some second topic", + first_topic: topic(@first_topic_hex_string_1), + second_topic: topic(@second_topic_hex_string_1), block_number: block.number ] log2_details = [ address: contract_address, transaction: transaction, - first_topic: "some OTHER first topic", - second_topic: "some OTHER second topic", + first_topic: topic(@first_topic_hex_string_2), + second_topic: topic(@second_topic_hex_string_2), block_number: block.number ] @@ -409,27 +428,27 @@ defmodule Explorer.Etherscan.LogsTest do log1_details = [ address: contract_address, transaction: transaction, - first_topic: "some first topic", - second_topic: "some second topic", - third_topic: "some third topic", + first_topic: topic(@first_topic_hex_string_1), + second_topic: topic(@second_topic_hex_string_1), + third_topic: topic(@third_topic_hex_string_1), block_number: block.number ] log2_details = [ address: contract_address, transaction: transaction, - first_topic: "some OTHER first topic", - second_topic: "some OTHER second topic", - third_topic: "some OTHER third topic", + first_topic: topic(@first_topic_hex_string_2), + second_topic: topic(@second_topic_hex_string_2), + third_topic: topic(@third_topic_hex_string_2), block_number: block.number ] log3_details = [ address: contract_address, transaction: transaction, - first_topic: "some ALT first topic", - second_topic: "some ALT second topic", - third_topic: "some ALT third topic", + first_topic: topic(@first_topic_hex_string_3), + second_topic: topic(@second_topic_hex_string_3), + third_topic: topic(@third_topic_hex_string_3), block_number: block.number ] @@ -469,27 +488,27 @@ defmodule Explorer.Etherscan.LogsTest do log1_details = [ address: contract_address, transaction: transaction, - first_topic: "some first topic", - second_topic: "some second topic", - third_topic: "some third topic", + first_topic: topic(@first_topic_hex_string_1), + second_topic: topic(@second_topic_hex_string_1), + third_topic: topic(@third_topic_hex_string_1), block_number: block.number ] log2_details = [ address: contract_address, transaction: transaction, - first_topic: "some OTHER first topic", - second_topic: "some OTHER second topic", - third_topic: "some OTHER third topic", + first_topic: topic(@first_topic_hex_string_2), + second_topic: topic(@second_topic_hex_string_2), + third_topic: topic(@third_topic_hex_string_2), block_number: block.number ] log3_details = [ address: contract_address, transaction: transaction, - first_topic: "some ALT first topic", - second_topic: "some ALT second topic", - third_topic: "some ALT third topic", + first_topic: topic(@first_topic_hex_string_3), + second_topic: topic(@second_topic_hex_string_3), + third_topic: topic(@third_topic_hex_string_3), block_number: block.number ] @@ -529,27 +548,27 @@ defmodule Explorer.Etherscan.LogsTest do log1_details = [ address: contract_address, transaction: transaction, - first_topic: "some topic", - second_topic: "some second topic", - third_topic: "some third topic", + first_topic: topic(@first_topic_hex_string_1), + second_topic: topic(@second_topic_hex_string_1), + third_topic: topic(@third_topic_hex_string_1), block_number: block.number ] log2_details = [ address: contract_address, transaction: transaction, - first_topic: "some topic", - second_topic: "some OTHER second topic", - third_topic: "some third topic", + first_topic: topic(@first_topic_hex_string_1), + second_topic: topic(@second_topic_hex_string_2), + third_topic: topic(@third_topic_hex_string_1), block_number: block.number ] log3_details = [ address: contract_address, transaction: transaction, - first_topic: "some topic", - second_topic: "some second topic", - third_topic: "some third topic", + first_topic: topic(@first_topic_hex_string_1), + second_topic: topic(@second_topic_hex_string_1), + third_topic: topic(@third_topic_hex_string_1), block_number: block.number ] @@ -589,28 +608,28 @@ defmodule Explorer.Etherscan.LogsTest do log1_details = [ address: contract_address, transaction: transaction, - first_topic: "some topic", - second_topic: "some second topic", + first_topic: topic(@first_topic_hex_string_1), + second_topic: topic(@second_topic_hex_string_1), block_number: block.number ] log2_details = [ address: contract_address, transaction: transaction, - first_topic: "some OTHER topic", - second_topic: "some OTHER second topic", - third_topic: "some OTHER third topic", - fourth_topic: "some fourth topic", + first_topic: topic(@first_topic_hex_string_2), + second_topic: topic(@second_topic_hex_string_2), + third_topic: topic(@third_topic_hex_string_2), + fourth_topic: topic(@fourth_topic_hex_string_1), block_number: block.number ] log3_details = [ address: contract_address, transaction: transaction, - first_topic: "some topic", - second_topic: "some second topic", - third_topic: "some third topic", - fourth_topic: "some fourth topic", + first_topic: topic(@first_topic_hex_string_1), + second_topic: topic(@second_topic_hex_string_1), + third_topic: topic(@third_topic_hex_string_1), + fourth_topic: topic(@fourth_topic_hex_string_1), block_number: block.number ] diff --git a/apps/indexer/lib/indexer/fetcher/polygon_edge/deposit_execute.ex b/apps/indexer/lib/indexer/fetcher/polygon_edge/deposit_execute.ex index 4e2dd57d09..ef1ad37e4a 100644 --- a/apps/indexer/lib/indexer/fetcher/polygon_edge/deposit_execute.ex +++ b/apps/indexer/lib/indexer/fetcher/polygon_edge/deposit_execute.ex @@ -12,6 +12,7 @@ defmodule Indexer.Fetcher.PolygonEdge.DepositExecute do import EthereumJSONRPC, only: [quantity_to_integer: 1] import Indexer.Fetcher.PolygonEdge, only: [fill_block_range: 5, get_block_number_by_tag: 3] + import Indexer.Helper, only: [log_topic_to_string: 1] alias Explorer.{Chain, Repo} alias Explorer.Chain.Log @@ -128,8 +129,13 @@ defmodule Indexer.Fetcher.PolygonEdge.DepositExecute do @spec event_to_deposit_execute(binary(), binary(), binary(), binary()) :: map() def event_to_deposit_execute(second_topic, third_topic, l2_transaction_hash, l2_block_number) do + msg_id = + second_topic + |> log_topic_to_string() + |> quantity_to_integer() + %{ - msg_id: quantity_to_integer(second_topic), + msg_id: msg_id, l2_transaction_hash: l2_transaction_hash, l2_block_number: quantity_to_integer(l2_block_number), success: quantity_to_integer(third_topic) != 0 diff --git a/apps/indexer/lib/indexer/fetcher/polygon_edge/withdrawal.ex b/apps/indexer/lib/indexer/fetcher/polygon_edge/withdrawal.ex index 80fb20568c..098545140a 100644 --- a/apps/indexer/lib/indexer/fetcher/polygon_edge/withdrawal.ex +++ b/apps/indexer/lib/indexer/fetcher/polygon_edge/withdrawal.ex @@ -13,6 +13,7 @@ defmodule Indexer.Fetcher.PolygonEdge.Withdrawal do import EthereumJSONRPC, only: [quantity_to_integer: 1] import Explorer.Helper, only: [decode_data: 2] import Indexer.Fetcher.PolygonEdge, only: [fill_block_range: 5, get_block_number_by_tag: 3] + import Indexer.Helper, only: [log_topic_to_string: 1] alias ABI.TypeDecoder alias Explorer.{Chain, Repo} @@ -133,6 +134,11 @@ defmodule Indexer.Fetcher.PolygonEdge.Withdrawal do @spec event_to_withdrawal(binary(), map(), binary(), binary()) :: map() def event_to_withdrawal(second_topic, data, l2_transaction_hash, l2_block_number) do + msg_id = + second_topic + |> log_topic_to_string() + |> quantity_to_integer() + [data_bytes] = decode_data(data, [:bytes]) sig = binary_part(data_bytes, 0, 32) @@ -148,7 +154,7 @@ defmodule Indexer.Fetcher.PolygonEdge.Withdrawal do end %{ - msg_id: quantity_to_integer(second_topic), + msg_id: msg_id, from: from, to: to, l2_transaction_hash: l2_transaction_hash, diff --git a/apps/indexer/lib/indexer/helper.ex b/apps/indexer/lib/indexer/helper.ex index 540606fe22..21b69831f2 100644 --- a/apps/indexer/lib/indexer/helper.ex +++ b/apps/indexer/lib/indexer/helper.ex @@ -32,4 +32,13 @@ defmodule Indexer.Helper do def is_address_correct?(_address) do false end + + @spec log_topic_to_string(any()) :: binary() | nil + def log_topic_to_string(topic) do + if is_binary(topic) or is_nil(topic) do + topic + else + Hash.to_string(topic) + end + end end diff --git a/apps/indexer/lib/indexer/temporary/uncataloged_token_transfers.ex b/apps/indexer/lib/indexer/temporary/uncataloged_token_transfers.ex index 17f676f136..d048a33311 100644 --- a/apps/indexer/lib/indexer/temporary/uncataloged_token_transfers.ex +++ b/apps/indexer/lib/indexer/temporary/uncataloged_token_transfers.ex @@ -12,7 +12,7 @@ defmodule Indexer.Temporary.UncatalogedTokenTransfers do require Logger - alias Explorer.Chain + alias Explorer.Chain.TokenTransfer alias Indexer.Block.Catchup.Fetcher alias Indexer.Temporary.UncatalogedTokenTransfers @@ -52,7 +52,7 @@ defmodule Indexer.Temporary.UncatalogedTokenTransfers do end def handle_info(:scan, state) do - {:ok, block_numbers} = Chain.uncataloged_token_transfer_block_numbers() + {:ok, block_numbers} = TokenTransfer.uncataloged_token_transfer_block_numbers() case block_numbers do [] -> diff --git a/apps/indexer/lib/indexer/transform/transaction_actions.ex b/apps/indexer/lib/indexer/transform/transaction_actions.ex index 1620fd724d..618c311f2a 100644 --- a/apps/indexer/lib/indexer/transform/transaction_actions.ex +++ b/apps/indexer/lib/indexer/transform/transaction_actions.ex @@ -285,8 +285,15 @@ defmodule Indexer.Transform.TransactionActions do [debt_amount, collateral_amount, _liquidator, _receive_a_token] = decode_data(log.data, [{:uint, 256}, {:uint, 256}, :address, :bool]) - debt_address = truncate_address_hash(log.third_topic) - collateral_address = truncate_address_hash(log.second_topic) + debt_address = + log.third_topic + |> Helper.log_topic_to_string() + |> truncate_address_hash() + + collateral_address = + log.second_topic + |> Helper.log_topic_to_string() + |> truncate_address_hash() case get_token_data([debt_address, collateral_address]) do false -> @@ -318,7 +325,10 @@ defmodule Indexer.Transform.TransactionActions do defp aave_handle_event(type, amount, log, address_topic, chain_id) when type in ["borrow", "supply", "withdraw", "repay", "flash_loan"] do - address = truncate_address_hash(address_topic) + address = + address_topic + |> Helper.log_topic_to_string() + |> truncate_address_hash() case get_token_data([address]) do false -> @@ -345,7 +355,10 @@ defmodule Indexer.Transform.TransactionActions do end defp aave_handle_event(type, log, address_topic, chain_id) when type in ["enable_collateral", "disable_collateral"] do - address = truncate_address_hash(address_topic) + address = + address_topic + |> Helper.log_topic_to_string() + |> truncate_address_hash() case get_token_data([address]) do false -> @@ -448,12 +461,23 @@ defmodule Indexer.Transform.TransactionActions do |> Enum.reduce(%{}, fn log, acc -> if sanitize_first_topic(log.first_topic) == @uniswap_v3_transfer_nft_event do # This is Transfer event for NFT - from = truncate_address_hash(log.second_topic) + from = + log.second_topic + |> Helper.log_topic_to_string() + |> truncate_address_hash() # credo:disable-for-next-line if from == burn_address_hash_string() do - to = truncate_address_hash(log.third_topic) - [token_id] = decode_data(log.fourth_topic, [{:uint, 256}]) + to = + log.third_topic + |> Helper.log_topic_to_string() + |> truncate_address_hash() + + [token_id] = + log.fourth_topic + |> Helper.log_topic_to_string() + |> decode_data([{:uint, 256}]) + mint_nft_ids = Map.put_new(acc, to, %{ids: [], log_index: log.index}) Map.put(mint_nft_ids, to, %{ @@ -970,7 +994,7 @@ defmodule Indexer.Transform.TransactionActions do end defp sanitize_first_topic(first_topic) do - if is_nil(first_topic), do: "", else: String.downcase(first_topic) + if is_nil(first_topic), do: "", else: String.downcase(Helper.log_topic_to_string(first_topic)) end defp truncate_address_hash(nil), do: burn_address_hash_string() From 83978978f9d565e49dcfc4f6a6cc5f39d153b397 Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Thu, 21 Dec 2023 18:56:28 +0300 Subject: [PATCH 22/33] Drop logs type column and related index (#9007) * Drop logs type index * Remove log type column * Update CHANGE LOG entry --- CHANGELOG.md | 1 + .../channels/websocket_v2_test.exs | 9 ++-- .../lib/ethereum_jsonrpc/log.ex | 43 ++++++------------- .../lib/ethereum_jsonrpc/receipts.ex | 15 +++---- .../test/ethereum_jsonrpc/receipts_test.exs | 6 +-- .../lib/explorer/chain/import/runner/logs.ex | 6 +-- apps/explorer/lib/explorer/chain/log.ex | 15 ++----- apps/explorer/lib/explorer/eth_rpc.ex | 3 +- apps/explorer/lib/explorer/etherscan/logs.ex | 3 +- ...1215115638_drop_unused_logs_type_index.exs | 11 +++++ .../test/explorer/chain/import_test.exs | 4 +- apps/explorer/test/explorer/chain_test.exs | 4 +- apps/explorer/test/support/factory.ex | 3 +- .../lib/indexer/transform/mint_transfers.ex | 3 +- .../indexer/block/fetcher/receipts_test.exs | 6 +-- .../test/indexer/block/fetcher_test.exs | 3 +- .../indexer/transform/mint_transfers_test.exs | 6 +-- .../transform/token_transfers_test.exs | 39 ++++++----------- 18 files changed, 65 insertions(+), 115 deletions(-) create mode 100644 apps/explorer/priv/repo/migrations/20231215115638_drop_unused_logs_type_index.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index f852bfdf77..48ef1d03c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ ### Chore - [#9009](https://github.com/blockscout/blockscout/pull/9009) - Index for block refetch_needed +- [#9007](https://github.com/blockscout/blockscout/pull/9007) - Drop logs type index - [#9006](https://github.com/blockscout/blockscout/pull/9006) - Drop unused indexes on address_current_token_balances table - [#9000](https://github.com/blockscout/blockscout/pull/9000) - Change log topic type in the DB to bytea - [#8996](https://github.com/blockscout/blockscout/pull/8996) - Refine token transfers token ids index diff --git a/apps/block_scout_web/test/block_scout_web/channels/websocket_v2_test.exs b/apps/block_scout_web/test/block_scout_web/channels/websocket_v2_test.exs index 2e06c53dbd..6d7aad632f 100644 --- a/apps/block_scout_web/test/block_scout_web/channels/websocket_v2_test.exs +++ b/apps/block_scout_web/test/block_scout_web/channels/websocket_v2_test.exs @@ -47,8 +47,7 @@ defmodule BlockScoutWeb.WebsocketV2Test do third_topic: third_topic, fourth_topic: nil, index: 0, - transaction_hash: "0x53bd884872de3e488692881baeec262e7b95234d3965248c39fe992fffd433e5", - type: "mined" + transaction_hash: "0x53bd884872de3e488692881baeec262e7b95234d3965248c39fe992fffd433e5" }, %{ block_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd", @@ -59,8 +58,7 @@ defmodule BlockScoutWeb.WebsocketV2Test do third_topic: third_topic, fourth_topic: nil, index: 1, - transaction_hash: "0x53bd884872de3e488692881baeec262e7b95234d3965248c39fe992fffd433e5", - type: "mined" + transaction_hash: "0x53bd884872de3e488692881baeec262e7b95234d3965248c39fe992fffd433e5" }, %{ block_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd", @@ -71,8 +69,7 @@ defmodule BlockScoutWeb.WebsocketV2Test do third_topic: third_topic, fourth_topic: nil, index: 2, - transaction_hash: "0x53bd884872de3e488692881baeec262e7b95234d3965248c39fe992fffd433e5", - type: "mined" + transaction_hash: "0x53bd884872de3e488692881baeec262e7b95234d3965248c39fe992fffd433e5" } ], timeout: 5 diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/log.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/log.ex index ecda583644..f3c6a88662 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/log.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/log.ex @@ -33,8 +33,7 @@ defmodule EthereumJSONRPC.Log do ...> "topics" => ["0x600bcf04a13e752d1e3670a5a9f1c21177ca2a93c6f5391d4f1298d098097c22"], ...> "transactionHash" => "0x53bd884872de3e488692881baeec262e7b95234d3965248c39fe992fffd433e5", ...> "transactionIndex" => 0, - ...> "transactionLogIndex" => 0, - ...> "type" => "mined" + ...> "transactionLogIndex" => 0 ...> } ...> ) %{ @@ -47,12 +46,9 @@ defmodule EthereumJSONRPC.Log do index: 0, second_topic: nil, third_topic: nil, - transaction_hash: "0x53bd884872de3e488692881baeec262e7b95234d3965248c39fe992fffd433e5", - type: "mined" + transaction_hash: "0x53bd884872de3e488692881baeec262e7b95234d3965248c39fe992fffd433e5" } - Geth does not supply a `"type"` - iex> EthereumJSONRPC.Log.elixir_to_params( ...> %{ ...> "address" => "0xda8b3276cde6d768a44b9dac659faa339a41ac55", @@ -82,17 +78,15 @@ defmodule EthereumJSONRPC.Log do } """ - def elixir_to_params( - %{ - "address" => address_hash, - "blockNumber" => block_number, - "blockHash" => block_hash, - "data" => data, - "logIndex" => index, - "topics" => topics, - "transactionHash" => transaction_hash - } = elixir - ) do + def elixir_to_params(%{ + "address" => address_hash, + "blockNumber" => block_number, + "blockHash" => block_hash, + "data" => data, + "logIndex" => index, + "topics" => topics, + "transactionHash" => transaction_hash + }) do %{ address_hash: address_hash, block_number: block_number, @@ -102,7 +96,6 @@ defmodule EthereumJSONRPC.Log do transaction_hash: transaction_hash } |> put_topics(topics) - |> put_type(elixir) end @doc """ @@ -118,8 +111,7 @@ defmodule EthereumJSONRPC.Log do ...> "topics" => ["0x600bcf04a13e752d1e3670a5a9f1c21177ca2a93c6f5391d4f1298d098097c22"], ...> "transactionHash" => "0x53bd884872de3e488692881baeec262e7b95234d3965248c39fe992fffd433e5", ...> "transactionIndex" => "0x0", - ...> "transactionLogIndex" => "0x0", - ...> "type" => "mined" + ...> "transactionLogIndex" => "0x0" ...> } ...> ) %{ @@ -131,8 +123,7 @@ defmodule EthereumJSONRPC.Log do "topics" => ["0x600bcf04a13e752d1e3670a5a9f1c21177ca2a93c6f5391d4f1298d098097c22"], "transactionHash" => "0x53bd884872de3e488692881baeec262e7b95234d3965248c39fe992fffd433e5", "transactionIndex" => 0, - "transactionLogIndex" => 0, - "type" => "mined" + "transactionLogIndex" => 0 } Geth includes a `"removed"` key @@ -172,7 +163,7 @@ defmodule EthereumJSONRPC.Log do end defp entry_to_elixir({key, _} = entry) - when key in ~w(address blockHash data removed topics transactionHash type timestamp), + when key in ~w(address blockHash data removed topics transactionHash timestamp), do: entry defp entry_to_elixir({key, quantity}) when key in ~w(blockNumber logIndex transactionIndex transactionLogIndex) do @@ -190,10 +181,4 @@ defmodule EthereumJSONRPC.Log do |> Map.put(:third_topic, Enum.at(topics, 2)) |> Map.put(:fourth_topic, Enum.at(topics, 3)) end - - defp put_type(params, %{"type" => type}) do - Map.put(params, :type, type) - end - - defp put_type(params, _), do: params end diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/receipts.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/receipts.ex index 8e0cffa7f2..37cc0a963c 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/receipts.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/receipts.ex @@ -32,8 +32,7 @@ defmodule EthereumJSONRPC.Receipts do ...> "topics" => ["0x600bcf04a13e752d1e3670a5a9f1c21177ca2a93c6f5391d4f1298d098097c22"], ...> "transactionHash" => "0x53bd884872de3e488692881baeec262e7b95234d3965248c39fe992fffd433e5", ...> "transactionIndex" => 0, - ...> "transactionLogIndex" => 0, - ...> "type" => "mined" + ...> "transactionLogIndex" => 0 ...> } ...> ], ...> "logsBloom" => "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000400000000000000000000000000000000000000000000000200000000000000000000020000000000000000200000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", @@ -53,8 +52,7 @@ defmodule EthereumJSONRPC.Receipts do "topics" => ["0x600bcf04a13e752d1e3670a5a9f1c21177ca2a93c6f5391d4f1298d098097c22"], "transactionHash" => "0x53bd884872de3e488692881baeec262e7b95234d3965248c39fe992fffd433e5", "transactionIndex" => 0, - "transactionLogIndex" => 0, - "type" => "mined" + "transactionLogIndex" => 0 } ] @@ -84,8 +82,7 @@ defmodule EthereumJSONRPC.Receipts do ...> "topics" => ["0x600bcf04a13e752d1e3670a5a9f1c21177ca2a93c6f5391d4f1298d098097c22"], ...> "transactionHash" => "0x53bd884872de3e488692881baeec262e7b95234d3965248c39fe992fffd433e5", ...> "transactionIndex" => 0, - ...> "transactionLogIndex" => 0, - ...> "type" => "mined" + ...> "transactionLogIndex" => 0 ...> } ...> ], ...> "logsBloom" => "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000400000000000000000000000000000000000000000000000200000000000000000000020000000000000000200000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", @@ -165,8 +162,7 @@ defmodule EthereumJSONRPC.Receipts do ...> "topics" => ["0x600bcf04a13e752d1e3670a5a9f1c21177ca2a93c6f5391d4f1298d098097c22"], ...> "transactionHash" => "0x53bd884872de3e488692881baeec262e7b95234d3965248c39fe992fffd433e5", ...> "transactionIndex" => "0x0", - ...> "transactionLogIndex" => "0x0", - ...> "type" => "mined" + ...> "transactionLogIndex" => "0x0" ...> } ...> ], ...> "logsBloom" => "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000400000000000000000000000000000000000000000000000200000000000000000000020000000000000000200000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", @@ -193,8 +189,7 @@ defmodule EthereumJSONRPC.Receipts do "topics" => ["0x600bcf04a13e752d1e3670a5a9f1c21177ca2a93c6f5391d4f1298d098097c22"], "transactionHash" => "0x53bd884872de3e488692881baeec262e7b95234d3965248c39fe992fffd433e5", "transactionIndex" => 0, - "transactionLogIndex" => 0, - "type" => "mined" + "transactionLogIndex" => 0 } ], "logsBloom" => "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000400000000000000000000000000000000000000000000000200000000000000000000020000000000000000200000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", diff --git a/apps/ethereum_jsonrpc/test/ethereum_jsonrpc/receipts_test.exs b/apps/ethereum_jsonrpc/test/ethereum_jsonrpc/receipts_test.exs index f554c31ce5..945f3f78bc 100644 --- a/apps/ethereum_jsonrpc/test/ethereum_jsonrpc/receipts_test.exs +++ b/apps/ethereum_jsonrpc/test/ethereum_jsonrpc/receipts_test.exs @@ -23,7 +23,6 @@ defmodule EthereumJSONRPC.ReceiptsTest do index: index, first_topic: first_topic, status: status, - type: type, transaction_hash: transaction_hash, transaction_index: transaction_index } = @@ -41,7 +40,6 @@ defmodule EthereumJSONRPC.ReceiptsTest do first_topic: "0xf6db2bace4ac8277384553ad9603d045220a91fb2448ab6130d7a6f044f9a8cf", gas_used: 106_025, status: nil, - type: nil, transaction_hash: "0xd3efddbbeb6ad8d8bb3f6b8c8fb6165567e9dd868013146bdbeb60953c82822a", transaction_index: 17 } @@ -58,7 +56,6 @@ defmodule EthereumJSONRPC.ReceiptsTest do index: 0, first_topic: "0x600bcf04a13e752d1e3670a5a9f1c21177ca2a93c6f5391d4f1298d098097c22", status: :ok, - type: "mined", transaction_hash: "0x53bd884872de3e488692881baeec262e7b95234d3965248c39fe992fffd433e5", transaction_index: 0 } @@ -89,8 +86,7 @@ defmodule EthereumJSONRPC.ReceiptsTest do "data" => data, "logIndex" => integer_to_quantity(index), "topics" => [first_topic], - "transactionHash" => transaction_hash, - "type" => type + "transactionHash" => transaction_hash } ], "status" => native_status, diff --git a/apps/explorer/lib/explorer/chain/import/runner/logs.ex b/apps/explorer/lib/explorer/chain/import/runner/logs.ex index c90adaa04a..7c0e591a0f 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/logs.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/logs.ex @@ -92,7 +92,6 @@ defmodule Explorer.Chain.Import.Runner.Logs do third_topic: fragment("EXCLUDED.third_topic"), fourth_topic: fragment("EXCLUDED.fourth_topic"), # Don't update `index` as it is part of the composite primary key and used for the conflict target - type: fragment("EXCLUDED.type"), # Don't update `transaction_hash` as it is part of the composite primary key and used for the conflict target inserted_at: fragment("LEAST(?, EXCLUDED.inserted_at)", log.inserted_at), updated_at: fragment("GREATEST(?, EXCLUDED.updated_at)", log.updated_at) @@ -100,14 +99,13 @@ defmodule Explorer.Chain.Import.Runner.Logs do ], where: fragment( - "(EXCLUDED.address_hash, EXCLUDED.data, EXCLUDED.first_topic, EXCLUDED.second_topic, EXCLUDED.third_topic, EXCLUDED.fourth_topic, EXCLUDED.type) IS DISTINCT FROM (?, ?, ?, ?, ?, ?, ?)", + "(EXCLUDED.address_hash, EXCLUDED.data, EXCLUDED.first_topic, EXCLUDED.second_topic, EXCLUDED.third_topic, EXCLUDED.fourth_topic) IS DISTINCT FROM (?, ?, ?, ?, ?, ?)", log.address_hash, log.data, log.first_topic, log.second_topic, log.third_topic, - log.fourth_topic, - log.type + log.fourth_topic ) ) end diff --git a/apps/explorer/lib/explorer/chain/log.ex b/apps/explorer/lib/explorer/chain/log.ex index e1c87a8a1e..be27eead76 100644 --- a/apps/explorer/lib/explorer/chain/log.ex +++ b/apps/explorer/lib/explorer/chain/log.ex @@ -12,7 +12,7 @@ defmodule Explorer.Chain.Log do alias Explorer.SmartContract.SigProviderInterface @required_attrs ~w(address_hash data block_hash index transaction_hash)a - @optional_attrs ~w(first_topic second_topic third_topic fourth_topic type block_number)a + @optional_attrs ~w(first_topic second_topic third_topic fourth_topic block_number)a @typedoc """ * `address` - address of contract that generate the event @@ -27,7 +27,6 @@ defmodule Explorer.Chain.Log do * `transaction` - transaction for which `log` is * `transaction_hash` - foreign key for `transaction`. * `index` - index of the log entry in all logs for the `transaction` - * `type` - type of event. *Nethermind-only* """ @type t :: %__MODULE__{ address: %Ecto.Association.NotLoaded{} | Address.t(), @@ -41,8 +40,7 @@ defmodule Explorer.Chain.Log do fourth_topic: Hash.Full.t(), transaction: %Ecto.Association.NotLoaded{} | Transaction.t(), transaction_hash: Hash.Full.t(), - index: non_neg_integer(), - type: String.t() | nil + index: non_neg_integer() } @primary_key false @@ -53,7 +51,6 @@ defmodule Explorer.Chain.Log do field(:third_topic, Hash.Full) field(:fourth_topic, Hash.Full) field(:index, :integer, primary_key: true) - field(:type, :string) field(:block_number, :integer) timestamps() @@ -76,8 +73,7 @@ defmodule Explorer.Chain.Log do end @doc """ - `address_hash` and `transaction_hash` are converted to `t:Explorer.Chain.Hash.t/0`. The allowed values for `type` - are currently unknown, so it is left as a `t:String.t/0`. + `address_hash` and `transaction_hash` are converted to `t:Explorer.Chain.Hash.t/0`. iex> changeset = Explorer.Chain.Log.changeset( ...> %Explorer.Chain.Log{}, @@ -90,8 +86,7 @@ defmodule Explorer.Chain.Log do ...> index: 0, ...> second_topic: nil, ...> third_topic: nil, - ...> transaction_hash: "0x53bd884872de3e488692881baeec262e7b95234d3965248c39fe992fffd433e5", - ...> type: "mined" + ...> transaction_hash: "0x53bd884872de3e488692881baeec262e7b95234d3965248c39fe992fffd433e5" ...> } ...> ) iex> changeset.valid? @@ -107,8 +102,6 @@ defmodule Explorer.Chain.Log do bytes: <<83, 189, 136, 72, 114, 222, 62, 72, 134, 146, 136, 27, 174, 236, 38, 46, 123, 149, 35, 77, 57, 101, 36, 140, 57, 254, 153, 47, 255, 212, 51, 229>> } - iex> changeset.changes.type - "mined" """ def changeset(%__MODULE__{} = log, attrs \\ %{}) do diff --git a/apps/explorer/lib/explorer/eth_rpc.ex b/apps/explorer/lib/explorer/eth_rpc.ex index 0e5b493a9a..d7260cf398 100644 --- a/apps/explorer/lib/explorer/eth_rpc.ex +++ b/apps/explorer/lib/explorer/eth_rpc.ex @@ -180,8 +180,7 @@ defmodule Explorer.EthRPC do "topics" => topics, "transactionHash" => to_string(log.transaction_hash), "transactionIndex" => log.transaction_index, - "transactionLogIndex" => log.index, - "type" => "mined" + "transactionLogIndex" => log.index } end diff --git a/apps/explorer/lib/explorer/etherscan/logs.ex b/apps/explorer/lib/explorer/etherscan/logs.ex index e83587869d..a0c432ce44 100644 --- a/apps/explorer/lib/explorer/etherscan/logs.ex +++ b/apps/explorer/lib/explorer/etherscan/logs.ex @@ -34,8 +34,7 @@ defmodule Explorer.Etherscan.Logs do :fourth_topic, :index, :address_hash, - :transaction_hash, - :type + :transaction_hash ] @default_paging_options %{block_number: nil, log_index: nil} diff --git a/apps/explorer/priv/repo/migrations/20231215115638_drop_unused_logs_type_index.exs b/apps/explorer/priv/repo/migrations/20231215115638_drop_unused_logs_type_index.exs new file mode 100644 index 0000000000..fc7df4ddce --- /dev/null +++ b/apps/explorer/priv/repo/migrations/20231215115638_drop_unused_logs_type_index.exs @@ -0,0 +1,11 @@ +defmodule Explorer.Repo.Migrations.DropUnusedLogsTypeIndex do + use Ecto.Migration + + def change do + drop(index(:logs, [:type], name: :logs_type_index)) + + alter table(:logs) do + remove(:type) + end + end +end diff --git a/apps/explorer/test/explorer/chain/import_test.exs b/apps/explorer/test/explorer/chain/import_test.exs index a8d62bfcad..0d2fbfa02d 100644 --- a/apps/explorer/test/explorer/chain/import_test.exs +++ b/apps/explorer/test/explorer/chain/import_test.exs @@ -103,8 +103,7 @@ defmodule Explorer.Chain.ImportTest do third_topic: third_topic, fourth_topic: nil, index: 0, - transaction_hash: "0x53bd884872de3e488692881baeec262e7b95234d3965248c39fe992fffd433e5", - type: "mined" + transaction_hash: "0x53bd884872de3e488692881baeec262e7b95234d3965248c39fe992fffd433e5" } ], timeout: 5 @@ -296,7 +295,6 @@ defmodule Explorer.Chain.ImportTest do <<83, 189, 136, 72, 114, 222, 62, 72, 134, 146, 136, 27, 174, 236, 38, 46, 123, 149, 35, 77, 57, 101, 36, 140, 57, 254, 153, 47, 255, 212, 51, 229>> }, - type: "mined", inserted_at: %{}, updated_at: %{} } diff --git a/apps/explorer/test/explorer/chain_test.exs b/apps/explorer/test/explorer/chain_test.exs index 083d50d0f6..fb529bb074 100644 --- a/apps/explorer/test/explorer/chain_test.exs +++ b/apps/explorer/test/explorer/chain_test.exs @@ -1311,8 +1311,7 @@ defmodule Explorer.ChainTest do third_topic: third_topic, fourth_topic: nil, index: 0, - transaction_hash: "0x53bd884872de3e488692881baeec262e7b95234d3965248c39fe992fffd433e5", - type: "mined" + transaction_hash: "0x53bd884872de3e488692881baeec262e7b95234d3965248c39fe992fffd433e5" } ] }, @@ -1490,7 +1489,6 @@ defmodule Explorer.ChainTest do <<83, 189, 136, 72, 114, 222, 62, 72, 134, 146, 136, 27, 174, 236, 38, 46, 123, 149, 35, 77, 57, 101, 36, 140, 57, 254, 153, 47, 255, 212, 51, 229>> }, - type: "mined", inserted_at: %{}, updated_at: %{} } diff --git a/apps/explorer/test/support/factory.ex b/apps/explorer/test/support/factory.ex index 4c218afb38..5fdd286ee0 100644 --- a/apps/explorer/test/support/factory.ex +++ b/apps/explorer/test/support/factory.ex @@ -679,8 +679,7 @@ defmodule Explorer.Factory do index: sequence("log_index", & &1), second_topic: nil, third_topic: nil, - transaction: build(:transaction), - type: sequence("0x") + transaction: build(:transaction) } end diff --git a/apps/indexer/lib/indexer/transform/mint_transfers.ex b/apps/indexer/lib/indexer/transform/mint_transfers.ex index e57a9841a7..c53dde7a53 100644 --- a/apps/indexer/lib/indexer/transform/mint_transfers.ex +++ b/apps/indexer/lib/indexer/transform/mint_transfers.ex @@ -24,8 +24,7 @@ defmodule Indexer.Transform.MintTransfers do ...> index: 1, ...> second_topic: "0x0000000000000000000000009a4a90e2732f3fa4087b0bb4bf85c76d14833df1", ...> third_topic: "0x0000000000000000000000007301cfa0e1756b71869e93d4e4dca5c7d0eb0aa6", - ...> transaction_hash: "0x1d5066d30ff3404a9306733136103ac2b0b989951c38df637f464f3667f8d4ee", - ...> type: "mined" + ...> transaction_hash: "0x1d5066d30ff3404a9306733136103ac2b0b989951c38df637f464f3667f8d4ee" ...> } ...> ]) %{ diff --git a/apps/indexer/test/indexer/block/fetcher/receipts_test.exs b/apps/indexer/test/indexer/block/fetcher/receipts_test.exs index c70d10761a..a7c4510655 100644 --- a/apps/indexer/test/indexer/block/fetcher/receipts_test.exs +++ b/apps/indexer/test/indexer/block/fetcher/receipts_test.exs @@ -51,8 +51,7 @@ defmodule Indexer.Block.Fetcher.ReceiptsTest do "topics" => ["0x600bcf04a13e752d1e3670a5a9f1c21177ca2a93c6f5391d4f1298d098097c22"], "transactionHash" => "0x43bd884872de3e488692881baeec262e7b95234d3965248c39fe992fffd433e5", "transactionIndex" => "0x0", - "transactionLogIndex" => "0x0", - "type" => "mined" + "transactionLogIndex" => "0x0" } ], "logsBloom" => @@ -82,8 +81,7 @@ defmodule Indexer.Block.Fetcher.ReceiptsTest do "topics" => ["0x600bcf04a13e752d1e3670a5a9f1c21177ca2a93c6f5391d4f1298d098097c22"], "transactionHash" => "0x53bd884872de3e488692881baeec262e7b95234d3965248c39fe992fffd433e5", "transactionIndex" => "0x0", - "transactionLogIndex" => "0x0", - "type" => "mined" + "transactionLogIndex" => "0x0" } ], "logsBloom" => diff --git a/apps/indexer/test/indexer/block/fetcher_test.exs b/apps/indexer/test/indexer/block/fetcher_test.exs index 24be0dfa88..0498cfd37e 100644 --- a/apps/indexer/test/indexer/block/fetcher_test.exs +++ b/apps/indexer/test/indexer/block/fetcher_test.exs @@ -375,8 +375,7 @@ defmodule Indexer.Block.FetcherTest do "topics" => ["0x600bcf04a13e752d1e3670a5a9f1c21177ca2a93c6f5391d4f1298d098097c22"], "transactionHash" => "0x53bd884872de3e488692881baeec262e7b95234d3965248c39fe992fffd433e5", "transactionIndex" => "0x0", - "transactionLogIndex" => "0x0", - "type" => "mined" + "transactionLogIndex" => "0x0" } ], "logsBloom" => diff --git a/apps/indexer/test/indexer/transform/mint_transfers_test.exs b/apps/indexer/test/indexer/transform/mint_transfers_test.exs index 04499a7af4..4670e983a2 100644 --- a/apps/indexer/test/indexer/transform/mint_transfers_test.exs +++ b/apps/indexer/test/indexer/transform/mint_transfers_test.exs @@ -17,8 +17,7 @@ defmodule Indexer.Transform.MintTransfersTest do index: 1, second_topic: "0x0000000000000000000000009a4a90e2732f3fa4087b0bb4bf85c76d14833df1", third_topic: "0x0000000000000000000000007301cfa0e1756b71869e93d4e4dca5c7d0eb0aa6", - transaction_hash: "0x1d5066d30ff3404a9306733136103ac2b0b989951c38df637f464f3667f8d4ee", - type: "mined" + transaction_hash: "0x1d5066d30ff3404a9306733136103ac2b0b989951c38df637f464f3667f8d4ee" } ] @@ -47,8 +46,7 @@ defmodule Indexer.Transform.MintTransfersTest do index: 1, second_topic: "0x0000000000000000000000009a4a90e2732f3fa4087b0bb4bf85c76d14833df1", third_topic: "0x0000000000000000000000007301cfa0e1756b71869e93d4e4dca5c7d0eb0aa6", - transaction_hash: "0x1d5066d30ff3404a9306733136103ac2b0b989951c38df637f464f3667f8d4ee", - type: "mined" + transaction_hash: "0x1d5066d30ff3404a9306733136103ac2b0b989951c38df637f464f3667f8d4ee" } ] diff --git a/apps/indexer/test/indexer/transform/token_transfers_test.exs b/apps/indexer/test/indexer/transform/token_transfers_test.exs index 8833474acc..0c670035cc 100644 --- a/apps/indexer/test/indexer/transform/token_transfers_test.exs +++ b/apps/indexer/test/indexer/transform/token_transfers_test.exs @@ -19,8 +19,7 @@ defmodule Indexer.Transform.TokenTransfersTest do index: 8, second_topic: "0x000000000000000000000000556813d9cc20acfe8388af029a679d34a63388db", third_topic: "0x00000000000000000000000092148dd870fa1b7c4700f2bd7f44238821c26f73", - transaction_hash: "0x43dfd761974e8c3351d285ab65bee311454eb45b149a015fe7804a33252f19e5", - type: "mined" + transaction_hash: "0x43dfd761974e8c3351d285ab65bee311454eb45b149a015fe7804a33252f19e5" }, %{ address_hash: "0x6ea5ec9cb832e60b6b1654f5826e9be638f276a5", @@ -32,8 +31,7 @@ defmodule Indexer.Transform.TokenTransfersTest do index: 0, second_topic: "0x00000000000000000000000063b0595bb7a0b7edd0549c9557a0c8aee6da667b", third_topic: "0x000000000000000000000000f3089e15d0c23c181d7f98b0878b560bfe193a1d", - transaction_hash: "0x8425a9b81a9bd1c64861110c1a453b84719cb0361d6fa0db68abf7611b9a890e", - type: "mined" + transaction_hash: "0x8425a9b81a9bd1c64861110c1a453b84719cb0361d6fa0db68abf7611b9a890e" }, %{ address_hash: "0x91932e8c6776fb2b04abb71874a7988747728bb2", @@ -45,8 +43,7 @@ defmodule Indexer.Transform.TokenTransfersTest do index: 1, second_topic: "0x0000000000000000000000009851ba177554eb07271ac230a137551e6dd0aa84", third_topic: "0x000000000000000000000000dccb72afee70e60b0c1226288fe86c01b953e8ac", - transaction_hash: "0x4011d9a930a3da620321589a54dc0ca3b88216b4886c7a7c3aaad1fb17702d35", - type: "mined" + transaction_hash: "0x4011d9a930a3da620321589a54dc0ca3b88216b4886c7a7c3aaad1fb17702d35" }, %{ address_hash: "0x0BE9e53fd7EDaC9F859882AfdDa116645287C629", @@ -58,8 +55,7 @@ defmodule Indexer.Transform.TokenTransfersTest do third_topic: nil, fourth_topic: nil, index: 1, - transaction_hash: "0x185889bc91372106ecf114a4e23f4ee615e131ae3e698078bd5d2ed7e3f55a49", - type: "mined" + transaction_hash: "0x185889bc91372106ecf114a4e23f4ee615e131ae3e698078bd5d2ed7e3f55a49" }, %{ address_hash: "0x0BE9e53fd7EDaC9F859882AfdDa116645287C629", @@ -71,8 +67,7 @@ defmodule Indexer.Transform.TokenTransfersTest do third_topic: nil, fourth_topic: nil, index: 1, - transaction_hash: "0x07510dbfddbac9064f7d607c2d9a14aa26fa19cdfcd578c0b585ff2395df543f", - type: "mined" + transaction_hash: "0x07510dbfddbac9064f7d607c2d9a14aa26fa19cdfcd578c0b585ff2395df543f" } ] @@ -157,8 +152,7 @@ defmodule Indexer.Transform.TokenTransfersTest do second_topic: nil, third_topic: nil, transaction_hash: "0x6d2dd62c178e55a13b65601f227c4ffdd8aa4e3bcb1f24731363b4f7619e92c8", - block_hash: "0x79594150677f083756a37eee7b97ed99ab071f502104332cb3835bac345711ca", - type: "mined" + block_hash: "0x79594150677f083756a37eee7b97ed99ab071f502104332cb3835bac345711ca" } expected = %{ @@ -198,8 +192,7 @@ defmodule Indexer.Transform.TokenTransfersTest do fourth_topic: "0x0000000000000000000000009c978f4cfa1fe13406bcc05baf26a35716f881dd", index: 2, transaction_hash: "0x6d2dd62c178e55a13b65601f227c4ffdd8aa4e3bcb1f24731363b4f7619e92c8", - block_hash: "0x79594150677f083756a37eee7b97ed99ab071f502104332cb3835bac345711ca", - type: "mined" + block_hash: "0x79594150677f083756a37eee7b97ed99ab071f502104332cb3835bac345711ca" } assert TokenTransfers.parse([log]) == %{ @@ -240,8 +233,7 @@ defmodule Indexer.Transform.TokenTransfersTest do fourth_topic: "0x0000000000000000000000006c943470780461b00783ad530a53913bd2c104d3", index: 2, transaction_hash: "0x6d2dd62c178e55a13b65601f227c4ffdd8aa4e3bcb1f24731363b4f7619e92c8", - block_hash: "0x79594150677f083756a37eee7b97ed99ab071f502104332cb3835bac345711ca", - type: "mined" + block_hash: "0x79594150677f083756a37eee7b97ed99ab071f502104332cb3835bac345711ca" } assert TokenTransfers.parse([log]) == %{ @@ -275,8 +267,7 @@ defmodule Indexer.Transform.TokenTransfersTest do fourth_topic: "0x0000000000000000000000000000000000000000", index: 6, transaction_hash: "0xa6ad6588edb4abd8ca45f30d2f026ba20b68a3002a5870dbd30cc3752568483b", - block_hash: "0x61b720e40f8c521edd77a52cabce556c18b18b198f78e361f310003386ff1f02", - type: "mined" + block_hash: "0x61b720e40f8c521edd77a52cabce556c18b18b198f78e361f310003386ff1f02" } assert TokenTransfers.parse([log]) == %{ @@ -296,8 +287,7 @@ defmodule Indexer.Transform.TokenTransfersTest do index: 2, second_topic: nil, third_topic: nil, - transaction_hash: "0x6d2dd62c178e55a13b65601f227c4ffdd8aa4e3bcb1f24731363b4f7619e92c8", - type: "mined" + transaction_hash: "0x6d2dd62c178e55a13b65601f227c4ffdd8aa4e3bcb1f24731363b4f7619e92c8" } error = capture_log(fn -> %{tokens: [], token_transfers: []} = TokenTransfers.parse([log]) end) @@ -319,8 +309,7 @@ defmodule Indexer.Transform.TokenTransfersTest do index: 8, second_topic: "0x000000000000000000000000556813d9cc20acfe8388af029a679d34a63388db", third_topic: "0x00000000000000000000000092148dd870fa1b7c4700f2bd7f44238821c26f73", - transaction_hash: "0x43dfd761974e8c3351d285ab65bee311454eb45b149a015fe7804a33252f19e5", - type: "mined" + transaction_hash: "0x43dfd761974e8c3351d285ab65bee311454eb45b149a015fe7804a33252f19e5" } assert %{ @@ -343,8 +332,7 @@ defmodule Indexer.Transform.TokenTransfersTest do index: 8, second_topic: "0x000000000000000000000000556813d9cc20acfe8388af029a679d34a63388db", third_topic: "0x00000000000000000000000092148dd870fa1b7c4700f2bd7f44238821c26f73", - transaction_hash: "0x43dfd761974e8c3351d285ab65bee311454eb45b149a015fe7804a33252f19e5", - type: "mined" + transaction_hash: "0x43dfd761974e8c3351d285ab65bee311454eb45b149a015fe7804a33252f19e5" }, %{ address_hash: contract_address_hash, @@ -357,8 +345,7 @@ defmodule Indexer.Transform.TokenTransfersTest do fourth_topic: "0x0000000000000000000000009c978f4cfa1fe13406bcc05baf26a35716f881dd", index: 2, transaction_hash: "0x43dfd761974e8c3351d285ab65bee311454eb45b149a015fe7804a33252f19e5", - block_hash: "0x79594150677f083756a37eee7b97ed99ab071f502104332cb3835bac345711ca", - type: "mined" + block_hash: "0x79594150677f083756a37eee7b97ed99ab071f502104332cb3835bac345711ca" } ] From a4799d35aa532461ebb8f05bfb81ae5a5f811b3e Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Thu, 21 Dec 2023 19:07:03 +0300 Subject: [PATCH 23/33] Drop unused token_id column from token_transfers table and indexes based on this column (#9005) * Drop unused token_id column from token_transfers table and indexes on this column * Remove token ids migration * Refactor DB migration * Update CHANGE LOG entry --- CHANGELOG.md | 1 + .../models/transaction_state_helper.ex | 2 +- .../lib/block_scout_web/schema/types.ex | 1 - .../tokens/inventory_controller_test.exs | 2 +- .../schema/query/token_transfers_test.exs | 3 - .../views/tokens/helper_test.exs | 2 +- apps/explorer/config/config.exs | 2 - apps/explorer/config/runtime/test.exs | 2 - apps/explorer/lib/explorer/application.ex | 3 +- .../lib/explorer/chain/token_transfer.ex | 5 +- .../chain/transaction/state_change.ex | 2 +- .../lowest_block_number_updater.ex | 65 -------------- .../supervisor.ex | 70 ---------------- .../worker.ex | 84 ------------------- ...ken_transfer_token_id_migrator_progress.ex | 67 --------------- ...5_drop_token_transfers_token_id_column.exs | 12 +++ .../lowest_block_number_updater_test.exs | 37 -------- .../worker_test.exs | 31 ------- 18 files changed, 19 insertions(+), 372 deletions(-) delete mode 100644 apps/explorer/lib/explorer/token_transfer_token_id_migration/lowest_block_number_updater.ex delete mode 100644 apps/explorer/lib/explorer/token_transfer_token_id_migration/supervisor.ex delete mode 100644 apps/explorer/lib/explorer/token_transfer_token_id_migration/worker.ex delete mode 100644 apps/explorer/lib/explorer/utility/token_transfer_token_id_migrator_progress.ex create mode 100644 apps/explorer/priv/repo/migrations/20231215094615_drop_token_transfers_token_id_column.exs delete mode 100644 apps/explorer/test/explorer/token_transfer_token_id_migration/lowest_block_number_updater_test.exs delete mode 100644 apps/explorer/test/explorer/token_transfer_token_id_migration/worker_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index 48ef1d03c8..7d3662a542 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ - [#9009](https://github.com/blockscout/blockscout/pull/9009) - Index for block refetch_needed - [#9007](https://github.com/blockscout/blockscout/pull/9007) - Drop logs type index - [#9006](https://github.com/blockscout/blockscout/pull/9006) - Drop unused indexes on address_current_token_balances table +- [#9005](https://github.com/blockscout/blockscout/pull/9005) - Drop unused token_id column from token_transfers table and indexes based on this column - [#9000](https://github.com/blockscout/blockscout/pull/9000) - Change log topic type in the DB to bytea - [#8996](https://github.com/blockscout/blockscout/pull/8996) - Refine token transfers token ids index - [#5322](https://github.com/blockscout/blockscout/pull/5322) - DB denormalization: block consensus and timestamp in transaction table diff --git a/apps/block_scout_web/lib/block_scout_web/models/transaction_state_helper.ex b/apps/block_scout_web/lib/block_scout_web/models/transaction_state_helper.ex index 3971a5463a..41b5bd1cfe 100644 --- a/apps/block_scout_web/lib/block_scout_web/models/transaction_state_helper.ex +++ b/apps/block_scout_web/lib/block_scout_web/models/transaction_state_helper.ex @@ -112,7 +112,7 @@ defmodule BlockScoutWeb.Models.TransactionStateHelper do token_ids = if token.type == "ERC-1155" do - token_transfer.token_ids || [token_transfer.token_id] + token_transfer.token_ids else [nil] end diff --git a/apps/block_scout_web/lib/block_scout_web/schema/types.ex b/apps/block_scout_web/lib/block_scout_web/schema/types.ex index 99f47a2916..d81f6eca51 100644 --- a/apps/block_scout_web/lib/block_scout_web/schema/types.ex +++ b/apps/block_scout_web/lib/block_scout_web/schema/types.ex @@ -130,7 +130,6 @@ defmodule BlockScoutWeb.Schema.Types do field(:amounts, list_of(:decimal)) field(:block_number, :integer) field(:log_index, :integer) - field(:token_id, :decimal) field(:token_ids, list_of(:decimal)) field(:from_address_hash, :address_hash) field(:to_address_hash, :address_hash) diff --git a/apps/block_scout_web/test/block_scout_web/controllers/tokens/inventory_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/tokens/inventory_controller_test.exs index 799b54da79..e15f855692 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/tokens/inventory_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/tokens/inventory_controller_test.exs @@ -126,7 +126,7 @@ defmodule BlockScoutWeb.Tokens.InventoryControllerTest do transaction: transaction, token_contract_address: token.contract_address, token: token, - token_id: 1000 + token_ids: [1000] ) conn = get(conn, token_inventory_path(conn, :index, token.contract_address_hash), %{type: "JSON"}) diff --git a/apps/block_scout_web/test/block_scout_web/schema/query/token_transfers_test.exs b/apps/block_scout_web/test/block_scout_web/schema/query/token_transfers_test.exs index ea20aa4f4e..d10a3fdcc6 100644 --- a/apps/block_scout_web/test/block_scout_web/schema/query/token_transfers_test.exs +++ b/apps/block_scout_web/test/block_scout_web/schema/query/token_transfers_test.exs @@ -16,7 +16,6 @@ defmodule BlockScoutWeb.Schema.Query.TokenTransfersTest do amounts block_number log_index - token_id token_ids from_address_hash to_address_hash @@ -45,7 +44,6 @@ defmodule BlockScoutWeb.Schema.Query.TokenTransfersTest do "amounts" => Enum.map(token_transfer.amounts, &to_string/1), "block_number" => token_transfer.block_number, "log_index" => token_transfer.log_index, - "token_id" => token_transfer.token_id, "token_ids" => Enum.map(token_transfer.token_ids, &to_string/1), "from_address_hash" => to_string(token_transfer.from_address_hash), "to_address_hash" => to_string(token_transfer.to_address_hash), @@ -70,7 +68,6 @@ defmodule BlockScoutWeb.Schema.Query.TokenTransfersTest do amount block_number log_index - token_id from_address_hash to_address_hash token_contract_address_hash diff --git a/apps/block_scout_web/test/block_scout_web/views/tokens/helper_test.exs b/apps/block_scout_web/test/block_scout_web/views/tokens/helper_test.exs index e59a85fb76..a8815379ce 100644 --- a/apps/block_scout_web/test/block_scout_web/views/tokens/helper_test.exs +++ b/apps/block_scout_web/test/block_scout_web/views/tokens/helper_test.exs @@ -27,7 +27,7 @@ defmodule BlockScoutWeb.Tokens.HelperTest do test "returns a string with the token_id with ERC-721 token" do token = build(:token, type: "ERC-721", decimals: nil) - token_transfer = build(:token_transfer, token: token, amount: nil, token_id: 1) + token_transfer = build(:token_transfer, token: token, amount: nil, token_ids: [1]) assert Helper.token_transfer_amount(token_transfer) == {:ok, :erc721_instance} end diff --git a/apps/explorer/config/config.exs b/apps/explorer/config/config.exs index 1dac8ff096..cc22af6be9 100644 --- a/apps/explorer/config/config.exs +++ b/apps/explorer/config/config.exs @@ -109,8 +109,6 @@ config :explorer, Explorer.Counters.BlockPriorityFeeCounter, enabled: true, enable_consolidation: true -config :explorer, Explorer.TokenTransferTokenIdMigration.Supervisor, enabled: true - config :explorer, Explorer.TokenInstanceOwnerAddressMigration.Supervisor, enabled: true config :explorer, Explorer.Migrator.TransactionsDenormalization, enabled: true diff --git a/apps/explorer/config/runtime/test.exs b/apps/explorer/config/runtime/test.exs index 367f4793f2..d496caa48a 100644 --- a/apps/explorer/config/runtime/test.exs +++ b/apps/explorer/config/runtime/test.exs @@ -33,8 +33,6 @@ config :explorer, Explorer.Market.History.Cataloger, enabled: false config :explorer, Explorer.Tracer, disabled?: false -config :explorer, Explorer.TokenTransferTokenIdMigration.Supervisor, enabled: false - config :explorer, Explorer.TokenInstanceOwnerAddressMigration.Supervisor, enabled: false config :explorer, Explorer.Migrator.TransactionsDenormalization, enabled: false diff --git a/apps/explorer/lib/explorer/application.ex b/apps/explorer/lib/explorer/application.ex index dfed4625c8..1749cb69fd 100644 --- a/apps/explorer/lib/explorer/application.ex +++ b/apps/explorer/lib/explorer/application.ex @@ -5,7 +5,7 @@ defmodule Explorer.Application do use Application - alias Explorer.{Admin, TokenTransferTokenIdMigration} + alias Explorer.Admin alias Explorer.Chain.Cache.{ Accounts, @@ -122,7 +122,6 @@ defmodule Explorer.Application do configure(Explorer.Validator.MetadataProcessor), configure(Explorer.Tags.AddressTag.Cataloger), configure(MinMissingBlockNumber), - configure(TokenTransferTokenIdMigration.Supervisor), configure(Explorer.Chain.Fetcher.CheckBytecodeMatchingOnDemand), configure(Explorer.Chain.Fetcher.FetchValidatorInfoOnDemand), configure(Explorer.TokenInstanceOwnerAddressMigration.Supervisor), diff --git a/apps/explorer/lib/explorer/chain/token_transfer.ex b/apps/explorer/lib/explorer/chain/token_transfer.ex index c53414e209..38dad61235 100644 --- a/apps/explorer/lib/explorer/chain/token_transfer.ex +++ b/apps/explorer/lib/explorer/chain/token_transfer.ex @@ -44,7 +44,6 @@ defmodule Explorer.Chain.TokenTransfer do * `:to_address_hash` - Address hash foreign key * `:token_contract_address` - The `t:Explorer.Chain.Address.t/0` of the token's contract. * `:token_contract_address_hash` - Address hash foreign key - * `:token_id` - ID of the token (applicable to ERC-721 tokens) * `:transaction` - The `t:Explorer.Chain.Transaction.t/0` ledger * `:transaction_hash` - Transaction foreign key * `:log_index` - Index of the corresponding `t:Explorer.Chain.Log.t/0` in the block. @@ -61,7 +60,6 @@ defmodule Explorer.Chain.TokenTransfer do to_address_hash: Hash.Address.t(), token_contract_address: %Ecto.Association.NotLoaded{} | Address.t(), token_contract_address_hash: Hash.Address.t(), - token_id: non_neg_integer() | nil, transaction: %Ecto.Association.NotLoaded{} | Transaction.t(), transaction_hash: Hash.Full.t(), log_index: non_neg_integer(), @@ -86,7 +84,6 @@ defmodule Explorer.Chain.TokenTransfer do field(:amount, :decimal) field(:block_number, :integer) field(:log_index, :integer, primary_key: true) - field(:token_id, :decimal) field(:amounts, {:array, :decimal}) field(:token_ids, {:array, :decimal}) field(:index_in_batch, :integer, virtual: true) @@ -129,7 +126,7 @@ defmodule Explorer.Chain.TokenTransfer do end @required_attrs ~w(block_number log_index from_address_hash to_address_hash token_contract_address_hash transaction_hash block_hash)a - @optional_attrs ~w(amount token_id amounts token_ids)a + @optional_attrs ~w(amount amounts token_ids)a @doc false def changeset(%TokenTransfer{} = struct, params \\ %{}) do diff --git a/apps/explorer/lib/explorer/chain/transaction/state_change.ex b/apps/explorer/lib/explorer/chain/transaction/state_change.ex index 3a399aa019..c0b70f23c4 100644 --- a/apps/explorer/lib/explorer/chain/transaction/state_change.ex +++ b/apps/explorer/lib/explorer/chain/transaction/state_change.ex @@ -120,7 +120,7 @@ defmodule Explorer.Chain.Transaction.StateChange do end defp do_update_balance(old_val, type, transfer, _) do - token_ids = if transfer.token.type == "ERC-1155", do: transfer.token_ids || [transfer.token_id], else: [nil] + token_ids = if transfer.token.type == "ERC-1155", do: transfer.token_ids, else: [nil] transfer_amounts = transfer.amounts || [transfer.amount || 1] sub_or_add = diff --git a/apps/explorer/lib/explorer/token_transfer_token_id_migration/lowest_block_number_updater.ex b/apps/explorer/lib/explorer/token_transfer_token_id_migration/lowest_block_number_updater.ex deleted file mode 100644 index 5794e524cc..0000000000 --- a/apps/explorer/lib/explorer/token_transfer_token_id_migration/lowest_block_number_updater.ex +++ /dev/null @@ -1,65 +0,0 @@ -defmodule Explorer.TokenTransferTokenIdMigration.LowestBlockNumberUpdater do - @moduledoc """ - Collects processed block numbers from token id migration workers - and updates last_processed_block_number according to them. - Full algorithm is in the 'Indexer.Fetcher.TokenTransferTokenIdMigration.Supervisor' module doc. - """ - use GenServer - - alias Explorer.Utility.TokenTransferTokenIdMigratorProgress - - def start_link(_) do - GenServer.start_link(__MODULE__, :ok, name: __MODULE__) - end - - @impl true - def init(_) do - last_processed_block_number = TokenTransferTokenIdMigratorProgress.get_last_processed_block_number() - - {:ok, %{last_processed_block_number: last_processed_block_number, processed_ranges: []}} - end - - def add_range(from, to) do - GenServer.cast(__MODULE__, {:add_range, from..to}) - end - - @impl true - def handle_cast({:add_range, range}, %{processed_ranges: processed_ranges} = state) do - ranges = - [range | processed_ranges] - |> Enum.sort_by(& &1.last, &>=/2) - |> normalize_ranges() - - {new_last_number, new_ranges} = maybe_update_last_processed_number(state.last_processed_block_number, ranges) - - {:noreply, %{last_processed_block_number: new_last_number, processed_ranges: new_ranges}} - end - - defp normalize_ranges(ranges) do - %{prev_range: prev, result: result} = - Enum.reduce(ranges, %{prev_range: nil, result: []}, fn range, %{prev_range: prev_range, result: result} -> - case {prev_range, range} do - {nil, _} -> - %{prev_range: range, result: result} - - {%{last: l1} = r1, %{first: f2} = r2} when l1 - 1 > f2 -> - %{prev_range: r2, result: [r1 | result]} - - {%{first: f1}, %{last: l2}} -> - %{prev_range: f1..l2, result: result} - end - end) - - Enum.reverse([prev | result]) - end - - # since ranges are normalized, we need to check only the first range to determine the new last_processed_number - defp maybe_update_last_processed_number(current_last, [from..to | rest] = ranges) when current_last - 1 <= from do - case TokenTransferTokenIdMigratorProgress.update_last_processed_block_number(to) do - {:ok, _} -> {to, rest} - _ -> {current_last, ranges} - end - end - - defp maybe_update_last_processed_number(current_last, ranges), do: {current_last, ranges} -end diff --git a/apps/explorer/lib/explorer/token_transfer_token_id_migration/supervisor.ex b/apps/explorer/lib/explorer/token_transfer_token_id_migration/supervisor.ex deleted file mode 100644 index 2121158fee..0000000000 --- a/apps/explorer/lib/explorer/token_transfer_token_id_migration/supervisor.ex +++ /dev/null @@ -1,70 +0,0 @@ -defmodule Explorer.TokenTransferTokenIdMigration.Supervisor do - @moduledoc """ - Supervises parts of token id migration process. - - Migration process algorithm: - - Defining the bounds of migration (by the first and the last block number of TokenTransfer). - Supervisor starts the workers in amount equal to 'TOKEN_ID_MIGRATION_CONCURRENCY' env value (defaults to 1) - and the 'LowestBlockNumberUpdater'. - - Each worker goes through the token transfers by batches ('TOKEN_ID_MIGRATION_BATCH_SIZE', defaults to 500) - and updates the token_ids to be equal of [token_id] for transfers that has any token_id. - Worker goes from the newest blocks to latest. - After worker is done with current batch, it sends the information about processed batch to 'LowestBlockNumberUpdater' - and takes the next by defining its bounds based on amount of all workers. - - For example, if batch size is 10, we have 5 workers and 100 items to be processed, - the distribution will be like this: - 1 worker - 99..90, 49..40 - 2 worker - 89..80, 39..30 - 3 worker - 79..70, 29..20 - 4 worker - 69..60, 19..10 - 5 worker - 59..50, 9..0 - - 'LowestBlockNumberUpdater' keeps the information about the last processed block number - (which is stored in the 'token_transfer_token_id_migrator_progress' db entity) - and block ranges that has already been processed by the workers but couldn't be committed - to last processed block number yet (because of the possible gap between the current last block - and upper bound of the last processed batch). Uncommitted block numbers are stored in normalize ranges. - When there is no gap between the last processed block number and the upper bound of the upper range, - 'LowestBlockNumberUpdater' updates the last processed block number in db and drops this range from its state. - - This supervisor won't start if the migration is completed - (last processed block number in db == 'TOKEN_ID_MIGRATION_FIRST_BLOCK' (defaults to 0)). - """ - use Supervisor - - alias Explorer.TokenTransferTokenIdMigration.{LowestBlockNumberUpdater, Worker} - alias Explorer.Utility.TokenTransferTokenIdMigratorProgress - - @default_first_block 0 - @default_workers_count 1 - - def start_link(_) do - Supervisor.start_link(__MODULE__, :ok, name: __MODULE__) - end - - @impl true - def init(_) do - first_block = Application.get_env(:explorer, :token_id_migration)[:first_block] || @default_first_block - last_block = TokenTransferTokenIdMigratorProgress.get_last_processed_block_number() - - if last_block > first_block do - workers_count = Application.get_env(:explorer, :token_id_migration)[:concurrency] || @default_workers_count - - workers = - Enum.map(1..workers_count, fn id -> - Supervisor.child_spec( - {Worker, idx: id, first_block: first_block, last_block: last_block, step: workers_count - 1}, - id: {Worker, id}, - restart: :transient - ) - end) - - Supervisor.init([LowestBlockNumberUpdater | workers], strategy: :one_for_one) - else - :ignore - end - end -end diff --git a/apps/explorer/lib/explorer/token_transfer_token_id_migration/worker.ex b/apps/explorer/lib/explorer/token_transfer_token_id_migration/worker.ex deleted file mode 100644 index f7916ed058..0000000000 --- a/apps/explorer/lib/explorer/token_transfer_token_id_migration/worker.ex +++ /dev/null @@ -1,84 +0,0 @@ -defmodule Explorer.TokenTransferTokenIdMigration.Worker do - @moduledoc """ - Performs the migration of TokenTransfer token_id to token_ids by batches. - Full algorithm is in the 'Explorer.TokenTransferTokenIdMigration.Supervisor' module doc. - """ - use GenServer - - import Ecto.Query - - alias Explorer.Chain.TokenTransfer - alias Explorer.Repo - alias Explorer.TokenTransferTokenIdMigration.LowestBlockNumberUpdater - - @default_batch_size 500 - @interval 10 - - def start_link(idx: idx, first_block: first, last_block: last, step: step) do - GenServer.start_link(__MODULE__, %{idx: idx, bottom_block: first, last_block: last, step: step}) - end - - @impl true - def init(%{idx: idx, bottom_block: bottom_block, last_block: last_block, step: step}) do - batch_size = Application.get_env(:explorer, :token_id_migration)[:batch_size] || @default_batch_size - range = calculate_new_range(last_block, bottom_block, batch_size, idx - 1) - - schedule_next_update() - - {:ok, %{batch_size: batch_size, bottom_block: bottom_block, step: step, current_range: range}} - end - - @impl true - def handle_info(:update, %{current_range: :out_of_bound} = state) do - {:stop, :normal, state} - end - - @impl true - def handle_info(:update, %{current_range: {lower_bound, upper_bound}} = state) do - case do_update(lower_bound, upper_bound) do - true -> - LowestBlockNumberUpdater.add_range(upper_bound, lower_bound) - new_range = calculate_new_range(lower_bound, state.bottom_block, state.batch_size, state.step) - schedule_next_update() - {:noreply, %{state | current_range: new_range}} - - _ -> - schedule_next_update() - {:noreply, state} - end - end - - defp calculate_new_range(last_processed_block, bottom_block, batch_size, step) do - upper_bound = last_processed_block - step * batch_size - 1 - lower_bound = max(upper_bound - batch_size + 1, bottom_block) - - if upper_bound >= bottom_block do - {lower_bound, upper_bound} - else - :out_of_bound - end - end - - defp do_update(lower_bound, upper_bound) do - token_transfers_batch_query = - from( - tt in TokenTransfer, - where: tt.block_number >= ^lower_bound, - where: tt.block_number <= ^upper_bound - ) - - token_transfers_batch_query - |> Repo.all() - |> Enum.filter(fn %{token_id: token_id} -> not is_nil(token_id) end) - |> Enum.map(fn token_transfer -> - token_transfer - |> TokenTransfer.changeset(%{token_ids: [token_transfer.token_id], token_id: nil}) - |> Repo.update() - end) - |> Enum.all?(&match?({:ok, _}, &1)) - end - - defp schedule_next_update do - Process.send_after(self(), :update, @interval) - end -end diff --git a/apps/explorer/lib/explorer/utility/token_transfer_token_id_migrator_progress.ex b/apps/explorer/lib/explorer/utility/token_transfer_token_id_migrator_progress.ex deleted file mode 100644 index 3a28181016..0000000000 --- a/apps/explorer/lib/explorer/utility/token_transfer_token_id_migrator_progress.ex +++ /dev/null @@ -1,67 +0,0 @@ -defmodule Explorer.Utility.TokenTransferTokenIdMigratorProgress do - @moduledoc """ - Module is responsible for keeping the current progress of TokenTransfer token_id migration. - Full algorithm is in the 'Indexer.Fetcher.TokenTransferTokenIdMigration.Supervisor' module doc. - """ - use Explorer.Schema - - require Logger - - alias Explorer.Chain.Cache.BlockNumber - alias Explorer.Repo - - schema "token_transfer_token_id_migrator_progress" do - field(:last_processed_block_number, :integer) - - timestamps() - end - - @doc false - def changeset(progress \\ %__MODULE__{}, params) do - cast(progress, params, [:last_processed_block_number]) - end - - def get_current_progress do - Repo.one( - from( - p in __MODULE__, - order_by: [desc: p.updated_at], - limit: 1 - ) - ) - end - - def get_last_processed_block_number do - case get_current_progress() do - nil -> - latest_processed_block_number = BlockNumber.get_max() + 1 - update_last_processed_block_number(latest_processed_block_number) - latest_processed_block_number - - %{last_processed_block_number: block_number} -> - block_number - end - end - - def update_last_processed_block_number(block_number, force \\ false) do - case get_current_progress() do - nil -> - %{last_processed_block_number: block_number} - |> changeset() - |> Repo.insert() - - progress -> - if not force and progress.last_processed_block_number < block_number do - Logger.error( - "TokenTransferTokenIdMigratorProgress new block_number is above the last one. Last: #{progress.last_processed_block_number}, new: #{block_number}" - ) - - {:error, :invalid_block_number} - else - progress - |> changeset(%{last_processed_block_number: block_number}) - |> Repo.update() - end - end - end -end diff --git a/apps/explorer/priv/repo/migrations/20231215094615_drop_token_transfers_token_id_column.exs b/apps/explorer/priv/repo/migrations/20231215094615_drop_token_transfers_token_id_column.exs new file mode 100644 index 0000000000..df8637071b --- /dev/null +++ b/apps/explorer/priv/repo/migrations/20231215094615_drop_token_transfers_token_id_column.exs @@ -0,0 +1,12 @@ +defmodule Explorer.Repo.Migrations.DropTokenTransfersTokenIdColumn do + use Ecto.Migration + + def change do + drop(index(:token_transfers, [:token_id])) + drop(index(:token_transfers, [:token_contract_address_hash, "token_id DESC", "block_number DESC"])) + + alter table(:token_transfers) do + remove(:token_id) + end + end +end diff --git a/apps/explorer/test/explorer/token_transfer_token_id_migration/lowest_block_number_updater_test.exs b/apps/explorer/test/explorer/token_transfer_token_id_migration/lowest_block_number_updater_test.exs deleted file mode 100644 index bdd94920db..0000000000 --- a/apps/explorer/test/explorer/token_transfer_token_id_migration/lowest_block_number_updater_test.exs +++ /dev/null @@ -1,37 +0,0 @@ -defmodule Explorer.TokenTransferTokenIdMigration.LowestBlockNumberUpdaterTest do - use Explorer.DataCase, async: false - - alias Explorer.Repo - alias Explorer.TokenTransferTokenIdMigration.LowestBlockNumberUpdater - alias Explorer.Utility.TokenTransferTokenIdMigratorProgress - - describe "Add range and update last processed block number" do - test "add_range/2" do - TokenTransferTokenIdMigratorProgress.update_last_processed_block_number(2000, true) - LowestBlockNumberUpdater.start_link([]) - - LowestBlockNumberUpdater.add_range(1000, 500) - LowestBlockNumberUpdater.add_range(1500, 1001) - Process.sleep(10) - - assert %{last_processed_block_number: 2000, processed_ranges: [1500..500//-1]} = - :sys.get_state(LowestBlockNumberUpdater) - - assert %{last_processed_block_number: 2000} = Repo.one(TokenTransferTokenIdMigratorProgress) - - LowestBlockNumberUpdater.add_range(499, 300) - LowestBlockNumberUpdater.add_range(299, 0) - Process.sleep(10) - - assert %{last_processed_block_number: 2000, processed_ranges: [1500..0//-1]} = - :sys.get_state(LowestBlockNumberUpdater) - - assert %{last_processed_block_number: 2000} = Repo.one(TokenTransferTokenIdMigratorProgress) - - LowestBlockNumberUpdater.add_range(1999, 1501) - Process.sleep(10) - assert %{last_processed_block_number: 0, processed_ranges: []} = :sys.get_state(LowestBlockNumberUpdater) - assert %{last_processed_block_number: 0} = Repo.one(TokenTransferTokenIdMigratorProgress) - end - end -end diff --git a/apps/explorer/test/explorer/token_transfer_token_id_migration/worker_test.exs b/apps/explorer/test/explorer/token_transfer_token_id_migration/worker_test.exs deleted file mode 100644 index 8797e90130..0000000000 --- a/apps/explorer/test/explorer/token_transfer_token_id_migration/worker_test.exs +++ /dev/null @@ -1,31 +0,0 @@ -defmodule Explorer.TokenTransferTokenIdMigration.WorkerTest do - use Explorer.DataCase, async: false - - alias Explorer.Repo - alias Explorer.TokenTransferTokenIdMigration.{LowestBlockNumberUpdater, Worker} - alias Explorer.Utility.TokenTransferTokenIdMigratorProgress - - describe "Move TokenTransfer token_id to token_ids" do - test "Move token_ids and update last processed block number" do - insert(:token_transfer, block_number: 1, token_id: 1, transaction: insert(:transaction)) - insert(:token_transfer, block_number: 500, token_id: 2, transaction: insert(:transaction)) - insert(:token_transfer, block_number: 1000, token_id: 3, transaction: insert(:transaction)) - insert(:token_transfer, block_number: 1500, token_id: 4, transaction: insert(:transaction)) - insert(:token_transfer, block_number: 2000, token_id: 5, transaction: insert(:transaction)) - - TokenTransferTokenIdMigratorProgress.update_last_processed_block_number(3000, true) - LowestBlockNumberUpdater.start_link([]) - - Worker.start_link(idx: 1, first_block: 0, last_block: 3000, step: 2) - Worker.start_link(idx: 2, first_block: 0, last_block: 3000, step: 2) - Worker.start_link(idx: 3, first_block: 0, last_block: 3000, step: 2) - Process.sleep(200) - - token_transfers = Repo.all(Explorer.Chain.TokenTransfer) - assert Enum.all?(token_transfers, fn tt -> is_nil(tt.token_id) end) - - expected_token_ids = [[Decimal.new(1)], [Decimal.new(2)], [Decimal.new(3)], [Decimal.new(4)], [Decimal.new(5)]] - assert ^expected_token_ids = token_transfers |> Enum.map(& &1.token_ids) |> Enum.sort_by(&List.first/1) - end - end -end From a7a3d2827b92d0fe5c416576ab9b79cc0b80a442 Mon Sep 17 00:00:00 2001 From: Qwerty5Uiop Date: Thu, 21 Dec 2023 14:06:14 +0600 Subject: [PATCH 24/33] Token type filling migrations --- apps/explorer/config/runtime/test.exs | 2 + apps/explorer/lib/explorer/application.ex | 4 +- .../explorer/chain/address/token_balance.ex | 29 +++++-- .../chain/cache/background_migrations.ex | 26 +++++- ...ddress_current_token_balance_token_type.ex | 51 ++++++++++++ .../address_token_balance_token_type.ex | 51 ++++++++++++ .../explorer/migrator/filling_migration.ex | 83 +++++++++++++++++++ .../migrator/transactions_denormalization.ex | 76 ++++------------- .../chain/address/token_balance_test.exs | 1 + ..._current_token_balance_token_type_test.exs | 32 +++++++ .../address_token_balance_token_type_test.exs | 32 +++++++ ...sactions_denormalization_migrator_test.exs | 7 +- 12 files changed, 320 insertions(+), 74 deletions(-) create mode 100644 apps/explorer/lib/explorer/migrator/address_current_token_balance_token_type.ex create mode 100644 apps/explorer/lib/explorer/migrator/address_token_balance_token_type.ex create mode 100644 apps/explorer/lib/explorer/migrator/filling_migration.ex create mode 100644 apps/explorer/test/explorer/migrator/address_current_token_balance_token_type_test.exs create mode 100644 apps/explorer/test/explorer/migrator/address_token_balance_token_type_test.exs diff --git a/apps/explorer/config/runtime/test.exs b/apps/explorer/config/runtime/test.exs index d496caa48a..ec346c1d3f 100644 --- a/apps/explorer/config/runtime/test.exs +++ b/apps/explorer/config/runtime/test.exs @@ -36,6 +36,8 @@ config :explorer, Explorer.Tracer, disabled?: false config :explorer, Explorer.TokenInstanceOwnerAddressMigration.Supervisor, enabled: false config :explorer, Explorer.Migrator.TransactionsDenormalization, enabled: false +config :explorer, Explorer.Migrator.AddressCurrentTokenBalanceTokenType, enabled: false +config :explorer, Explorer.Migrator.AddressTokenBalanceTokenType, enabled: false config :explorer, realtime_events_sender: Explorer.Chain.Events.SimpleSender diff --git a/apps/explorer/lib/explorer/application.ex b/apps/explorer/lib/explorer/application.ex index 1749cb69fd..52b196489d 100644 --- a/apps/explorer/lib/explorer/application.ex +++ b/apps/explorer/lib/explorer/application.ex @@ -127,7 +127,9 @@ defmodule Explorer.Application do configure(Explorer.TokenInstanceOwnerAddressMigration.Supervisor), sc_microservice_configure(Explorer.Chain.Fetcher.LookUpSmartContractSourcesOnDemand), configure(Explorer.Chain.Cache.RootstockLockedBTC), - configure(Explorer.Migrator.TransactionsDenormalization) + configure(Explorer.Migrator.TransactionsDenormalization), + configure(Explorer.Migrator.AddressCurrentTokenBalanceTokenType), + configure(Explorer.Migrator.AddressTokenBalanceTokenType) ] |> List.flatten() diff --git a/apps/explorer/lib/explorer/chain/address/token_balance.ex b/apps/explorer/lib/explorer/chain/address/token_balance.ex index ab7a75b624..5b647bae1e 100644 --- a/apps/explorer/lib/explorer/chain/address/token_balance.ex +++ b/apps/explorer/lib/explorer/chain/address/token_balance.ex @@ -13,6 +13,7 @@ defmodule Explorer.Chain.Address.TokenBalance do alias Explorer.Chain alias Explorer.Chain.Address.TokenBalance + alias Explorer.Chain.Cache.BackgroundMigrations alias Explorer.Chain.{Address, Block, Hash, Token} @typedoc """ @@ -80,15 +81,27 @@ defmodule Explorer.Chain.Address.TokenBalance do ignores the burn_address for tokens ERC-721 since the most tokens ERC-721 don't allow get the balance for burn_address. """ + # credo:disable-for-next-line /Complexity/ def unfetched_token_balances do - from( - tb in TokenBalance, - join: t in Token, - on: tb.token_contract_address_hash == t.contract_address_hash, - where: - ((tb.address_hash != ^@burn_address_hash and t.type == "ERC-721") or t.type == "ERC-20" or t.type == "ERC-1155") and - (is_nil(tb.value_fetched_at) or is_nil(tb.value)) - ) + if BackgroundMigrations.get_tb_token_type_finished() do + from( + tb in TokenBalance, + where: + ((tb.address_hash != ^@burn_address_hash and tb.token_type == "ERC-721") or tb.token_type == "ERC-20" or + tb.token_type == "ERC-1155") and + (is_nil(tb.value_fetched_at) or is_nil(tb.value)) + ) + else + from( + tb in TokenBalance, + join: t in Token, + on: tb.token_contract_address_hash == t.contract_address_hash, + where: + ((tb.address_hash != ^@burn_address_hash and t.type == "ERC-721") or t.type == "ERC-20" or + t.type == "ERC-1155") and + (is_nil(tb.value_fetched_at) or is_nil(tb.value)) + ) + end end @doc """ diff --git a/apps/explorer/lib/explorer/chain/cache/background_migrations.ex b/apps/explorer/lib/explorer/chain/cache/background_migrations.ex index 5e3b352457..3470f48c08 100644 --- a/apps/explorer/lib/explorer/chain/cache/background_migrations.ex +++ b/apps/explorer/lib/explorer/chain/cache/background_migrations.ex @@ -7,11 +7,17 @@ defmodule Explorer.Chain.Cache.BackgroundMigrations do use Explorer.Chain.MapCache, name: :background_migrations_status, - key: :denormalization_finished + key: :denormalization_finished, + key: :tb_token_type_finished, + key: :ctb_token_type_finished @dialyzer :no_match - alias Explorer.Migrator.TransactionsDenormalization + alias Explorer.Migrator.{ + AddressCurrentTokenBalanceTokenType, + AddressTokenBalanceTokenType, + TransactionsDenormalization + } defp handle_fallback(:denormalization_finished) do Task.start(fn -> @@ -20,4 +26,20 @@ defmodule Explorer.Chain.Cache.BackgroundMigrations do {:return, false} end + + defp handle_fallback(:tb_token_type_finished) do + Task.start(fn -> + set_tb_token_type_finished(AddressTokenBalanceTokenType.migration_finished?()) + end) + + {:return, false} + end + + defp handle_fallback(:ctb_token_type_finished) do + Task.start(fn -> + set_ctb_token_type_finished(AddressCurrentTokenBalanceTokenType.migration_finished?()) + end) + + {:return, false} + end end diff --git a/apps/explorer/lib/explorer/migrator/address_current_token_balance_token_type.ex b/apps/explorer/lib/explorer/migrator/address_current_token_balance_token_type.ex new file mode 100644 index 0000000000..93226db73f --- /dev/null +++ b/apps/explorer/lib/explorer/migrator/address_current_token_balance_token_type.ex @@ -0,0 +1,51 @@ +defmodule Explorer.Migrator.AddressCurrentTokenBalanceTokenType do + @moduledoc """ + Fill empty token_type's for address_current_token_balances + """ + + use Explorer.Migrator.FillingMigration + + import Ecto.Query + + alias Explorer.Chain.Address.CurrentTokenBalance + alias Explorer.Chain.Cache.BackgroundMigrations + alias Explorer.Migrator.FillingMigration + alias Explorer.Repo + + @migration_name "ctb_token_type" + + @impl FillingMigration + def migration_name, do: @migration_name + + @impl FillingMigration + def last_unprocessed_identifiers do + limit = batch_size() * concurrency() + + unprocessed_data_query() + |> select([ctb], ctb.id) + |> limit(^limit) + |> Repo.all(timeout: :infinity) + end + + @impl FillingMigration + def unprocessed_data_query do + from(ctb in CurrentTokenBalance, where: is_nil(ctb.token_type)) + end + + @impl FillingMigration + def update_batch(token_balance_ids) do + query = + from(current_token_balance in CurrentTokenBalance, + join: token in assoc(current_token_balance, :token), + where: current_token_balance.id in ^token_balance_ids, + update: [set: [token_type: token.type]] + ) + + Repo.update_all(query, [], timeout: :infinity) + end + + @impl FillingMigration + def update_cache do + BackgroundMigrations.set_ctb_token_type_finished(true) + end +end diff --git a/apps/explorer/lib/explorer/migrator/address_token_balance_token_type.ex b/apps/explorer/lib/explorer/migrator/address_token_balance_token_type.ex new file mode 100644 index 0000000000..9427db73ed --- /dev/null +++ b/apps/explorer/lib/explorer/migrator/address_token_balance_token_type.ex @@ -0,0 +1,51 @@ +defmodule Explorer.Migrator.AddressTokenBalanceTokenType do + @moduledoc """ + Fill empty token_type's for address_token_balances + """ + + use Explorer.Migrator.FillingMigration + + import Ecto.Query + + alias Explorer.Chain.Address.TokenBalance + alias Explorer.Chain.Cache.BackgroundMigrations + alias Explorer.Migrator.FillingMigration + alias Explorer.Repo + + @migration_name "tb_token_type" + + @impl FillingMigration + def migration_name, do: @migration_name + + @impl FillingMigration + def last_unprocessed_identifiers do + limit = batch_size() * concurrency() + + unprocessed_data_query() + |> select([tb], tb.id) + |> limit(^limit) + |> Repo.all(timeout: :infinity) + end + + @impl FillingMigration + def unprocessed_data_query do + from(tb in TokenBalance, where: is_nil(tb.token_type)) + end + + @impl FillingMigration + def update_batch(token_balance_ids) do + query = + from(token_balance in TokenBalance, + join: token in assoc(token_balance, :token), + where: token_balance.id in ^token_balance_ids, + update: [set: [token_type: token.type]] + ) + + Repo.update_all(query, [], timeout: :infinity) + end + + @impl FillingMigration + def update_cache do + BackgroundMigrations.set_tb_token_type_finished(true) + end +end diff --git a/apps/explorer/lib/explorer/migrator/filling_migration.ex b/apps/explorer/lib/explorer/migrator/filling_migration.ex new file mode 100644 index 0000000000..05823e9517 --- /dev/null +++ b/apps/explorer/lib/explorer/migrator/filling_migration.ex @@ -0,0 +1,83 @@ +defmodule Explorer.Migrator.FillingMigration do + @moduledoc """ + Template for creating migrations that fills some fields in existing entities + """ + + @callback migration_name :: String.t() + @callback unprocessed_data_query :: Ecto.Query.t() + @callback last_unprocessed_identifiers :: [any()] + @callback update_batch([any()]) :: any() + @callback update_cache :: any() + + defmacro __using__(_opts) do + quote do + @behaviour Explorer.Migrator.FillingMigration + + use GenServer, restart: :transient + + import Ecto.Query + + alias Explorer.Migrator.MigrationStatus + alias Explorer.Repo + + @default_batch_size 500 + + def start_link(_) do + GenServer.start_link(__MODULE__, :ok, name: __MODULE__) + end + + def migration_finished? do + not Repo.exists?(unprocessed_data_query()) + end + + @impl true + def init(_) do + case MigrationStatus.get_status(migration_name()) do + "completed" -> + :ignore + + _ -> + MigrationStatus.set_status(migration_name(), "started") + schedule_batch_migration() + {:ok, %{}} + end + end + + @impl true + def handle_info(:migrate_batch, state) do + case last_unprocessed_identifiers() do + [] -> + update_cache() + MigrationStatus.set_status(migration_name(), "completed") + {:stop, :normal, state} + + hashes -> + hashes + |> Enum.chunk_every(batch_size()) + |> Enum.map(&run_task/1) + |> Task.await_many(:infinity) + + schedule_batch_migration() + + {:noreply, state} + end + end + + defp run_task(batch), do: Task.async(fn -> update_batch(batch) end) + + defp schedule_batch_migration do + Process.send(self(), :migrate_batch, []) + end + + defp batch_size do + Application.get_env(:explorer, __MODULE__)[:batch_size] || @default_batch_size + end + + defp concurrency do + default = 4 * System.schedulers_online() + + Application.get_env(:explorer, __MODULE__)[:concurrency] || default + end + end + end +end diff --git a/apps/explorer/lib/explorer/migrator/transactions_denormalization.ex b/apps/explorer/lib/explorer/migrator/transactions_denormalization.ex index 166d7f8340..fe66548aba 100644 --- a/apps/explorer/lib/explorer/migrator/transactions_denormalization.ex +++ b/apps/explorer/lib/explorer/migrator/transactions_denormalization.ex @@ -3,78 +3,39 @@ defmodule Explorer.Migrator.TransactionsDenormalization do Migrates all transactions to have set block_consensus and block_timestamp """ - use GenServer, restart: :transient + use Explorer.Migrator.FillingMigration import Ecto.Query alias Explorer.Chain.Cache.BackgroundMigrations alias Explorer.Chain.Transaction - alias Explorer.Migrator.MigrationStatus + alias Explorer.Migrator.FillingMigration alias Explorer.Repo - @default_batch_size 500 @migration_name "denormalization" - @spec start_link(term()) :: GenServer.on_start() - def start_link(_) do - GenServer.start_link(__MODULE__, :ok, name: __MODULE__) - end - - def migration_finished? do - not Repo.exists?(unprocessed_transactions_query()) - end - - @impl true - def init(_) do - case MigrationStatus.get_status(@migration_name) do - "completed" -> - :ignore - - _ -> - MigrationStatus.set_status(@migration_name, "started") - schedule_batch_migration() - {:ok, %{}} - end - end - - @impl true - def handle_info(:migrate_batch, state) do - case last_unprocessed_transaction_hashes() do - [] -> - BackgroundMigrations.set_denormalization_finished(true) - MigrationStatus.set_status(@migration_name, "completed") - {:stop, :normal, state} - - hashes -> - hashes - |> Enum.chunk_every(batch_size()) - |> Enum.map(&run_task/1) - |> Task.await_many(:infinity) - - schedule_batch_migration() + @impl FillingMigration + def migration_name, do: @migration_name - {:noreply, state} - end - end - - defp run_task(batch), do: Task.async(fn -> update_batch(batch) end) - - defp last_unprocessed_transaction_hashes do + @impl FillingMigration + def last_unprocessed_identifiers do limit = batch_size() * concurrency() - unprocessed_transactions_query() + unprocessed_data_query() |> select([t], t.hash) |> limit(^limit) |> Repo.all(timeout: :infinity) end - defp unprocessed_transactions_query do + @impl FillingMigration + def unprocessed_data_query do from(t in Transaction, where: not is_nil(t.block_hash) and (is_nil(t.block_consensus) or is_nil(t.block_timestamp)) ) end - defp update_batch(transaction_hashes) do + @impl FillingMigration + def update_batch(transaction_hashes) do query = from(transaction in Transaction, join: block in assoc(transaction, :block), @@ -85,17 +46,8 @@ defmodule Explorer.Migrator.TransactionsDenormalization do Repo.update_all(query, [], timeout: :infinity) end - defp schedule_batch_migration do - Process.send(self(), :migrate_batch, []) - end - - defp batch_size do - Application.get_env(:explorer, __MODULE__)[:batch_size] || @default_batch_size - end - - defp concurrency do - default = 4 * System.schedulers_online() - - Application.get_env(:explorer, __MODULE__)[:concurrency] || default + @impl FillingMigration + def update_cache do + BackgroundMigrations.set_denormalization_finished(true) end end diff --git a/apps/explorer/test/explorer/chain/address/token_balance_test.exs b/apps/explorer/test/explorer/chain/address/token_balance_test.exs index 9a72837f6a..3e1a801828 100644 --- a/apps/explorer/test/explorer/chain/address/token_balance_test.exs +++ b/apps/explorer/test/explorer/chain/address/token_balance_test.exs @@ -46,6 +46,7 @@ defmodule Explorer.Chain.Address.TokenBalanceTest do :token_balance, address: burn_address, token_contract_address_hash: token.contract_address_hash, + token_type: "ERC-721", value_fetched_at: nil ) diff --git a/apps/explorer/test/explorer/migrator/address_current_token_balance_token_type_test.exs b/apps/explorer/test/explorer/migrator/address_current_token_balance_token_type_test.exs new file mode 100644 index 0000000000..f9e43e12e6 --- /dev/null +++ b/apps/explorer/test/explorer/migrator/address_current_token_balance_token_type_test.exs @@ -0,0 +1,32 @@ +defmodule Explorer.Migrator.AddressCurrentTokenBalanceTokenTypeTest do + use Explorer.DataCase, async: false + + alias Explorer.Chain.Cache.BackgroundMigrations + alias Explorer.Chain.Address.CurrentTokenBalance + alias Explorer.Migrator.{AddressCurrentTokenBalanceTokenType, MigrationStatus} + alias Explorer.Repo + + describe "Migrate current token balances" do + test "Set token_type for not processed current token balances" do + Enum.each(0..10, fn _x -> + current_token_balance = insert(:address_current_token_balance, token_type: nil) + assert %{token_type: nil} = current_token_balance + end) + + assert MigrationStatus.get_status("ctb_token_type") == nil + + AddressCurrentTokenBalanceTokenType.start_link([]) + Process.sleep(100) + + CurrentTokenBalance + |> Repo.all() + |> Repo.preload(:token) + |> Enum.each(fn ctb -> + assert %{token_type: token_type, token: %{type: token_type}} = ctb + assert not is_nil(token_type) + end) + + assert MigrationStatus.get_status("ctb_token_type") == "completed" + end + end +end diff --git a/apps/explorer/test/explorer/migrator/address_token_balance_token_type_test.exs b/apps/explorer/test/explorer/migrator/address_token_balance_token_type_test.exs new file mode 100644 index 0000000000..e1ac9d997f --- /dev/null +++ b/apps/explorer/test/explorer/migrator/address_token_balance_token_type_test.exs @@ -0,0 +1,32 @@ +defmodule Explorer.Migrator.AddressTokenBalanceTokenTypeTest do + use Explorer.DataCase, async: false + + alias Explorer.Chain.Cache.BackgroundMigrations + alias Explorer.Chain.Address.TokenBalance + alias Explorer.Migrator.{AddressTokenBalanceTokenType, MigrationStatus} + alias Explorer.Repo + + describe "Migrate token balances" do + test "Set token_type for not processed token balances" do + Enum.each(0..10, fn _x -> + token_balance = insert(:token_balance, token_type: nil) + assert %{token_type: nil} = token_balance + end) + + assert MigrationStatus.get_status("tb_token_type") == nil + + AddressTokenBalanceTokenType.start_link([]) + Process.sleep(100) + + TokenBalance + |> Repo.all() + |> Repo.preload(:token) + |> Enum.each(fn tb -> + assert %{token_type: token_type, token: %{type: token_type}} = tb + assert not is_nil(token_type) + end) + + assert MigrationStatus.get_status("tb_token_type") == "completed" + end + end +end diff --git a/apps/explorer/test/explorer/migrator/transactions_denormalization_migrator_test.exs b/apps/explorer/test/explorer/migrator/transactions_denormalization_migrator_test.exs index 8505844a79..e47afe96da 100644 --- a/apps/explorer/test/explorer/migrator/transactions_denormalization_migrator_test.exs +++ b/apps/explorer/test/explorer/migrator/transactions_denormalization_migrator_test.exs @@ -1,8 +1,9 @@ defmodule Explorer.Migrator.TransactionsDenormalizationTest do use Explorer.DataCase, async: false + alias Explorer.Chain.Cache.BackgroundMigrations alias Explorer.Chain.Transaction - alias Explorer.Migrator.TransactionsDenormalization + alias Explorer.Migrator.{MigrationStatus, TransactionsDenormalization} alias Explorer.Repo describe "Migrate transactions" do @@ -20,6 +21,8 @@ defmodule Explorer.Migrator.TransactionsDenormalizationTest do assert not is_nil(timestamp) end) + assert MigrationStatus.get_status("denormalization") == nil + TransactionsDenormalization.start_link([]) Process.sleep(100) @@ -33,6 +36,8 @@ defmodule Explorer.Migrator.TransactionsDenormalizationTest do block: %{consensus: consensus, timestamp: timestamp} } = t end) + + assert MigrationStatus.get_status("denormalization") == "completed" end end end From 7b5838c091bb70ccf9259fae97830c3cc1df46e4 Mon Sep 17 00:00:00 2001 From: Qwerty5Uiop Date: Mon, 25 Dec 2023 15:16:51 +0600 Subject: [PATCH 25/33] Enable migrators in config --- apps/explorer/config/config.exs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/explorer/config/config.exs b/apps/explorer/config/config.exs index cc22af6be9..eafec8dec7 100644 --- a/apps/explorer/config/config.exs +++ b/apps/explorer/config/config.exs @@ -112,6 +112,8 @@ config :explorer, Explorer.Counters.BlockPriorityFeeCounter, config :explorer, Explorer.TokenInstanceOwnerAddressMigration.Supervisor, enabled: true config :explorer, Explorer.Migrator.TransactionsDenormalization, enabled: true +config :explorer, Explorer.Migrator.AddressCurrentTokenBalanceTokenType, enabled: true +config :explorer, Explorer.Migrator.AddressTokenBalanceTokenType, enabled: true config :explorer, Explorer.Chain.Fetcher.CheckBytecodeMatchingOnDemand, enabled: true From a67da9a7e73de7d97d99f5b1ac1e2f3016f0c780 Mon Sep 17 00:00:00 2001 From: Qwerty5Uiop Date: Wed, 27 Dec 2023 16:49:42 +0600 Subject: [PATCH 26/33] Improve filling migrations tests --- .../migrator/address_current_token_balance_token_type_test.exs | 1 + .../explorer/migrator/address_token_balance_token_type_test.exs | 1 + 2 files changed, 2 insertions(+) diff --git a/apps/explorer/test/explorer/migrator/address_current_token_balance_token_type_test.exs b/apps/explorer/test/explorer/migrator/address_current_token_balance_token_type_test.exs index f9e43e12e6..27591d9c52 100644 --- a/apps/explorer/test/explorer/migrator/address_current_token_balance_token_type_test.exs +++ b/apps/explorer/test/explorer/migrator/address_current_token_balance_token_type_test.exs @@ -27,6 +27,7 @@ defmodule Explorer.Migrator.AddressCurrentTokenBalanceTokenTypeTest do end) assert MigrationStatus.get_status("ctb_token_type") == "completed" + assert BackgroundMigrations.get_ctb_token_type_finished() == true end end end diff --git a/apps/explorer/test/explorer/migrator/address_token_balance_token_type_test.exs b/apps/explorer/test/explorer/migrator/address_token_balance_token_type_test.exs index e1ac9d997f..4bf09c5712 100644 --- a/apps/explorer/test/explorer/migrator/address_token_balance_token_type_test.exs +++ b/apps/explorer/test/explorer/migrator/address_token_balance_token_type_test.exs @@ -27,6 +27,7 @@ defmodule Explorer.Migrator.AddressTokenBalanceTokenTypeTest do end) assert MigrationStatus.get_status("tb_token_type") == "completed" + assert BackgroundMigrations.get_tb_token_type_finished() == true end end end From 4f81bd8174c1132b41e3be6c150e30eb1e9d96db Mon Sep 17 00:00:00 2001 From: Viktor Baranov Date: Wed, 27 Dec 2023 13:58:38 +0300 Subject: [PATCH 27/33] Add CHANGELOG entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d3662a542..abed600601 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ ### Chore +- [#9038](https://github.com/blockscout/blockscout/pull/9038) - Token type filling migrations - [#9009](https://github.com/blockscout/blockscout/pull/9009) - Index for block refetch_needed - [#9007](https://github.com/blockscout/blockscout/pull/9007) - Drop logs type index - [#9006](https://github.com/blockscout/blockscout/pull/9006) - Drop unused indexes on address_current_token_balances table From c7f566156ee0376f562a12b48e7934564e7bbd6f Mon Sep 17 00:00:00 2001 From: Viktor Baranov Date: Mon, 25 Dec 2023 15:56:04 +0300 Subject: [PATCH 28/33] Add ASC indices for logs, token transfers, transactions --- CHANGELOG.md | 1 + ...0231225113850_transactions_asc_indices.exs | 47 +++++++++++++++++++ .../20231225115026_logs_asc_index.exs | 7 +++ ...231225115100_token_transfers_asc_index.exs | 7 +++ 4 files changed, 62 insertions(+) create mode 100644 apps/explorer/priv/repo/migrations/20231225113850_transactions_asc_indices.exs create mode 100644 apps/explorer/priv/repo/migrations/20231225115026_logs_asc_index.exs create mode 100644 apps/explorer/priv/repo/migrations/20231225115100_token_transfers_asc_index.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index abed600601..c1df85bc68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ ### Chore +- [#9055](https://github.com/blockscout/blockscout/pull/9055) - Add ASC indices for logs, token transfers, transactions - [#9038](https://github.com/blockscout/blockscout/pull/9038) - Token type filling migrations - [#9009](https://github.com/blockscout/blockscout/pull/9009) - Index for block refetch_needed - [#9007](https://github.com/blockscout/blockscout/pull/9007) - Drop logs type index diff --git a/apps/explorer/priv/repo/migrations/20231225113850_transactions_asc_indices.exs b/apps/explorer/priv/repo/migrations/20231225113850_transactions_asc_indices.exs new file mode 100644 index 0000000000..b0f668e89c --- /dev/null +++ b/apps/explorer/priv/repo/migrations/20231225113850_transactions_asc_indices.exs @@ -0,0 +1,47 @@ +defmodule Explorer.Repo.Migrations.TransactionsAscIndices do + use Ecto.Migration + + def change do + create( + index( + :transactions, + [ + :from_address_hash, + "block_number ASC NULLS LAST", + "index ASC NULLS LAST", + "inserted_at ASC", + "hash DESC" + ], + name: "transactions_from_address_hash_with_pending_index_asc" + ) + ) + + create( + index( + :transactions, + [ + :to_address_hash, + "block_number ASC NULLS LAST", + "index ASC NULLS LAST", + "inserted_at ASC", + "hash DESC" + ], + name: "transactions_to_address_hash_with_pending_index_asc" + ) + ) + + create( + index( + :transactions, + [ + :created_contract_address_hash, + "block_number ASC NULLS LAST", + "index ASC NULLS LAST", + "inserted_at ASC", + "hash DESC" + ], + name: "transactions_created_contract_address_hash_with_pending_index_a" + ) + ) + end +end diff --git a/apps/explorer/priv/repo/migrations/20231225115026_logs_asc_index.exs b/apps/explorer/priv/repo/migrations/20231225115026_logs_asc_index.exs new file mode 100644 index 0000000000..7e7fc0963d --- /dev/null +++ b/apps/explorer/priv/repo/migrations/20231225115026_logs_asc_index.exs @@ -0,0 +1,7 @@ +defmodule Explorer.Repo.Migrations.LogsAscIndex do + use Ecto.Migration + + def change do + create(index(:logs, ["block_number ASC, index ASC"])) + end +end diff --git a/apps/explorer/priv/repo/migrations/20231225115100_token_transfers_asc_index.exs b/apps/explorer/priv/repo/migrations/20231225115100_token_transfers_asc_index.exs new file mode 100644 index 0000000000..084ce83adb --- /dev/null +++ b/apps/explorer/priv/repo/migrations/20231225115100_token_transfers_asc_index.exs @@ -0,0 +1,7 @@ +defmodule Explorer.Repo.Migrations.TokenTransfersAscIndex do + use Ecto.Migration + + def change do + create(index(:token_transfers, ["block_number ASC", "log_index ASC"])) + end +end From 1cc481c98c2ecc33a4fb85a73143386092408efd Mon Sep 17 00:00:00 2001 From: Viktor Baranov Date: Wed, 27 Dec 2023 17:52:26 +0300 Subject: [PATCH 29/33] Invalidate GA cache --- .github/workflows/config.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/config.yml b/.github/workflows/config.yml index bd487d8c2a..82c11d6d85 100644 --- a/.github/workflows/config.yml +++ b/.github/workflows/config.yml @@ -75,7 +75,7 @@ jobs: path: | deps _build - key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_32-${{ hashFiles('mix.lock') }} + key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_33-${{ hashFiles('mix.lock') }} restore-keys: | ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps- @@ -133,7 +133,7 @@ jobs: path: | deps _build - key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_32-${{ hashFiles('mix.lock') }} + key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_33-${{ hashFiles('mix.lock') }} restore-keys: | ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-" @@ -157,7 +157,7 @@ jobs: path: | deps _build - key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_32-${{ hashFiles('mix.lock') }} + key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_33-${{ hashFiles('mix.lock') }} restore-keys: | ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-" @@ -186,7 +186,7 @@ jobs: path: | deps _build - key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_32-${{ hashFiles('mix.lock') }} + key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_33-${{ hashFiles('mix.lock') }} restore-keys: | ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-" @@ -230,7 +230,7 @@ jobs: path: | deps _build - key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_32-${{ hashFiles('mix.lock') }} + key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_33-${{ hashFiles('mix.lock') }} restore-keys: | ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-" @@ -256,7 +256,7 @@ jobs: path: | deps _build - key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_32-${{ hashFiles('mix.lock') }} + key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_33-${{ hashFiles('mix.lock') }} restore-keys: | ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-" @@ -285,7 +285,7 @@ jobs: path: | deps _build - key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_32-${{ hashFiles('mix.lock') }} + key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_33-${{ hashFiles('mix.lock') }} restore-keys: | ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-" @@ -333,7 +333,7 @@ jobs: path: | deps _build - key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_32-${{ hashFiles('mix.lock') }} + key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_33-${{ hashFiles('mix.lock') }} restore-keys: | ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-" @@ -379,7 +379,7 @@ jobs: path: | deps _build - key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_32-${{ hashFiles('mix.lock') }} + key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_33-${{ hashFiles('mix.lock') }} restore-keys: | ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-" @@ -441,7 +441,7 @@ jobs: path: | deps _build - key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_32-${{ hashFiles('mix.lock') }} + key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_33-${{ hashFiles('mix.lock') }} restore-keys: | ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-" @@ -501,7 +501,7 @@ jobs: path: | deps _build - key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_32-${{ hashFiles('mix.lock') }} + key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_33-${{ hashFiles('mix.lock') }} restore-keys: | ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-" @@ -572,7 +572,7 @@ jobs: path: | deps _build - key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_32-${{ hashFiles('mix.lock') }} + key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_33-${{ hashFiles('mix.lock') }} restore-keys: | ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-" @@ -640,7 +640,7 @@ jobs: path: | deps _build - key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_32-${{ hashFiles('mix.lock') }} + key: ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-mixlockhash_33-${{ hashFiles('mix.lock') }} restore-keys: | ${{ runner.os }}-${{ env.ELIXIR_VERSION }}-${{ env.OTP_VERSION }}-${{ env.MIX_ENV }}-deps-" From 286ddbc6f3cbc2dc2650aff4f78331fde54632d0 Mon Sep 17 00:00:00 2001 From: Viktor Baranov Date: Wed, 27 Dec 2023 18:22:09 +0300 Subject: [PATCH 30/33] Update version to 6.0.0 --- .github/ISSUE_TEMPLATE/bug_report.yml | 2 +- .github/workflows/prerelease.yml | 2 +- .../workflows/publish-docker-image-every-push.yml | 2 +- .../workflows/publish-docker-image-for-core.yml | 2 +- .../publish-docker-image-for-eth-goerli.yml | 2 +- .../publish-docker-image-for-eth-sepolia.yml | 2 +- .github/workflows/publish-docker-image-for-eth.yml | 2 +- .../workflows/publish-docker-image-for-fuse.yml | 2 +- .../publish-docker-image-for-immutable.yml | 2 +- .../publish-docker-image-for-l2-staging.yml | 2 +- .../workflows/publish-docker-image-for-lukso.yml | 2 +- .../publish-docker-image-for-optimism.yml | 2 +- .../publish-docker-image-for-polygon-edge.yml | 2 +- .github/workflows/publish-docker-image-for-rsk.yml | 2 +- .../publish-docker-image-for-stability.yml | 2 +- .../workflows/publish-docker-image-for-suave.yml | 2 +- .../workflows/publish-docker-image-for-xdai.yml | 2 +- .../workflows/publish-docker-image-for-zkevm.yml | 2 +- .../workflows/publish-docker-image-for-zksync.yml | 2 +- .../publish-docker-image-staging-on-demand.yml | 2 +- .github/workflows/release-additional.yml | 2 +- .github/workflows/release.yml | 2 +- CHANGELOG.md | 14 ++++++++++++++ apps/block_scout_web/mix.exs | 2 +- apps/ethereum_jsonrpc/mix.exs | 2 +- apps/explorer/mix.exs | 2 +- apps/indexer/mix.exs | 2 +- docker-compose/docker-compose.yml | 2 +- docker/Makefile | 2 +- mix.exs | 2 +- rel/config.exs | 2 +- 31 files changed, 44 insertions(+), 30 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 4f35712cae..dcd9fc29d2 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -65,7 +65,7 @@ body: attributes: label: Backend version description: The release version of the backend or branch/commit. - placeholder: v5.4.0 + placeholder: v6.0.0 validations: required: true diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index 070650c676..2d09956875 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -16,7 +16,7 @@ jobs: name: Push Docker image to Docker Hub runs-on: ubuntu-latest env: - RELEASE_VERSION: 5.4.0 + RELEASE_VERSION: 6.0.0 steps: - name: Check out the repo uses: actions/checkout@v4 diff --git a/.github/workflows/publish-docker-image-every-push.yml b/.github/workflows/publish-docker-image-every-push.yml index 10d2cfabc9..25d9c3fa99 100644 --- a/.github/workflows/publish-docker-image-every-push.yml +++ b/.github/workflows/publish-docker-image-every-push.yml @@ -11,7 +11,7 @@ on: env: OTP_VERSION: '25.2.1' ELIXIR_VERSION: '1.14.5' - RELEASE_VERSION: 5.4.0 + RELEASE_VERSION: 6.0.0 jobs: push_to_registry: diff --git a/.github/workflows/publish-docker-image-for-core.yml b/.github/workflows/publish-docker-image-for-core.yml index 0c76d12620..61cb6f8189 100644 --- a/.github/workflows/publish-docker-image-for-core.yml +++ b/.github/workflows/publish-docker-image-for-core.yml @@ -15,7 +15,7 @@ jobs: name: Push Docker image to Docker Hub runs-on: ubuntu-latest env: - RELEASE_VERSION: 5.4.0 + RELEASE_VERSION: 6.0.0 DOCKER_CHAIN_NAME: poa steps: - name: Check out the repo diff --git a/.github/workflows/publish-docker-image-for-eth-goerli.yml b/.github/workflows/publish-docker-image-for-eth-goerli.yml index 1911bbb357..f381283cf3 100644 --- a/.github/workflows/publish-docker-image-for-eth-goerli.yml +++ b/.github/workflows/publish-docker-image-for-eth-goerli.yml @@ -15,7 +15,7 @@ jobs: name: Push Docker image to Docker Hub runs-on: ubuntu-latest env: - RELEASE_VERSION: 5.4.0 + RELEASE_VERSION: 6.0.0 DOCKER_CHAIN_NAME: eth-goerli steps: - name: Check out the repo diff --git a/.github/workflows/publish-docker-image-for-eth-sepolia.yml b/.github/workflows/publish-docker-image-for-eth-sepolia.yml index 7c09456adc..b24c257167 100644 --- a/.github/workflows/publish-docker-image-for-eth-sepolia.yml +++ b/.github/workflows/publish-docker-image-for-eth-sepolia.yml @@ -15,7 +15,7 @@ jobs: name: Push Docker image to Docker Hub runs-on: ubuntu-latest env: - RELEASE_VERSION: 5.4.0 + RELEASE_VERSION: 6.0.0 DOCKER_CHAIN_NAME: eth-sepolia steps: - name: Check out the repo diff --git a/.github/workflows/publish-docker-image-for-eth.yml b/.github/workflows/publish-docker-image-for-eth.yml index f7a0ad5fd2..a663380306 100644 --- a/.github/workflows/publish-docker-image-for-eth.yml +++ b/.github/workflows/publish-docker-image-for-eth.yml @@ -15,7 +15,7 @@ jobs: name: Push Docker image to Docker Hub runs-on: ubuntu-latest env: - RELEASE_VERSION: 5.4.0 + RELEASE_VERSION: 6.0.0 DOCKER_CHAIN_NAME: mainnet steps: - name: Check out the repo diff --git a/.github/workflows/publish-docker-image-for-fuse.yml b/.github/workflows/publish-docker-image-for-fuse.yml index 286664969f..a9ec8713e7 100644 --- a/.github/workflows/publish-docker-image-for-fuse.yml +++ b/.github/workflows/publish-docker-image-for-fuse.yml @@ -15,7 +15,7 @@ jobs: name: Push Docker image to Docker Hub runs-on: ubuntu-latest env: - RELEASE_VERSION: 5.4.0 + RELEASE_VERSION: 6.0.0 DOCKER_CHAIN_NAME: fuse steps: - name: Check out the repo diff --git a/.github/workflows/publish-docker-image-for-immutable.yml b/.github/workflows/publish-docker-image-for-immutable.yml index 1c9e5cc85a..8bd44e3d1c 100644 --- a/.github/workflows/publish-docker-image-for-immutable.yml +++ b/.github/workflows/publish-docker-image-for-immutable.yml @@ -15,7 +15,7 @@ jobs: name: Push Docker image to Docker Hub runs-on: ubuntu-latest env: - RELEASE_VERSION: 5.4.0 + RELEASE_VERSION: 6.0.0 DOCKER_CHAIN_NAME: immutable steps: - name: Check out the repo diff --git a/.github/workflows/publish-docker-image-for-l2-staging.yml b/.github/workflows/publish-docker-image-for-l2-staging.yml index 637b880a2b..1c5f290907 100644 --- a/.github/workflows/publish-docker-image-for-l2-staging.yml +++ b/.github/workflows/publish-docker-image-for-l2-staging.yml @@ -15,7 +15,7 @@ jobs: name: Push Docker image to Docker Hub runs-on: ubuntu-latest env: - RELEASE_VERSION: 5.4.0 + RELEASE_VERSION: 6.0.0 DOCKER_CHAIN_NAME: optimism-l2-advanced steps: - name: Check out the repo diff --git a/.github/workflows/publish-docker-image-for-lukso.yml b/.github/workflows/publish-docker-image-for-lukso.yml index 4cd324babd..10efc69fe2 100644 --- a/.github/workflows/publish-docker-image-for-lukso.yml +++ b/.github/workflows/publish-docker-image-for-lukso.yml @@ -15,7 +15,7 @@ jobs: name: Push Docker image to Docker Hub runs-on: ubuntu-latest env: - RELEASE_VERSION: 5.4.0 + RELEASE_VERSION: 6.0.0 DOCKER_CHAIN_NAME: lukso steps: - name: Check out the repo diff --git a/.github/workflows/publish-docker-image-for-optimism.yml b/.github/workflows/publish-docker-image-for-optimism.yml index 320ed18ba4..f0c24fa582 100644 --- a/.github/workflows/publish-docker-image-for-optimism.yml +++ b/.github/workflows/publish-docker-image-for-optimism.yml @@ -15,7 +15,7 @@ jobs: name: Push Docker image to Docker Hub runs-on: ubuntu-latest env: - RELEASE_VERSION: 5.4.0 + RELEASE_VERSION: 6.0.0 DOCKER_CHAIN_NAME: optimism steps: - name: Check out the repo diff --git a/.github/workflows/publish-docker-image-for-polygon-edge.yml b/.github/workflows/publish-docker-image-for-polygon-edge.yml index d2153e9072..a7384c4f29 100644 --- a/.github/workflows/publish-docker-image-for-polygon-edge.yml +++ b/.github/workflows/publish-docker-image-for-polygon-edge.yml @@ -15,7 +15,7 @@ jobs: name: Push Docker image to Docker Hub runs-on: ubuntu-latest env: - RELEASE_VERSION: 5.4.0 + RELEASE_VERSION: 6.0.0 DOCKER_CHAIN_NAME: polygon-edge steps: - name: Check out the repo diff --git a/.github/workflows/publish-docker-image-for-rsk.yml b/.github/workflows/publish-docker-image-for-rsk.yml index 3f77608f0c..615ad8820b 100644 --- a/.github/workflows/publish-docker-image-for-rsk.yml +++ b/.github/workflows/publish-docker-image-for-rsk.yml @@ -15,7 +15,7 @@ jobs: name: Push Docker image to Docker Hub runs-on: ubuntu-latest env: - RELEASE_VERSION: 5.4.0 + RELEASE_VERSION: 6.0.0 DOCKER_CHAIN_NAME: rsk steps: - name: Check out the repo diff --git a/.github/workflows/publish-docker-image-for-stability.yml b/.github/workflows/publish-docker-image-for-stability.yml index b81cbc1334..c14e5a6a6a 100644 --- a/.github/workflows/publish-docker-image-for-stability.yml +++ b/.github/workflows/publish-docker-image-for-stability.yml @@ -18,7 +18,7 @@ jobs: name: Push Docker image to Docker Hub runs-on: ubuntu-latest env: - RELEASE_VERSION: 5.4.0 + RELEASE_VERSION: 6.0.0 DOCKER_CHAIN_NAME: stability steps: - name: Check out the repo diff --git a/.github/workflows/publish-docker-image-for-suave.yml b/.github/workflows/publish-docker-image-for-suave.yml index 8ba4ec4975..0f3c4ad439 100644 --- a/.github/workflows/publish-docker-image-for-suave.yml +++ b/.github/workflows/publish-docker-image-for-suave.yml @@ -18,7 +18,7 @@ jobs: name: Push Docker image to Docker Hub runs-on: ubuntu-latest env: - RELEASE_VERSION: 5.4.0 + RELEASE_VERSION: 6.0.0 DOCKER_CHAIN_NAME: suave steps: - name: Check out the repo diff --git a/.github/workflows/publish-docker-image-for-xdai.yml b/.github/workflows/publish-docker-image-for-xdai.yml index 7af9b838a3..b53fe18fc5 100644 --- a/.github/workflows/publish-docker-image-for-xdai.yml +++ b/.github/workflows/publish-docker-image-for-xdai.yml @@ -15,7 +15,7 @@ jobs: name: Push Docker image to Docker Hub runs-on: ubuntu-latest env: - RELEASE_VERSION: 5.4.0 + RELEASE_VERSION: 6.0.0 DOCKER_CHAIN_NAME: xdai steps: - name: Check out the repo diff --git a/.github/workflows/publish-docker-image-for-zkevm.yml b/.github/workflows/publish-docker-image-for-zkevm.yml index 603719db11..0e7072a394 100644 --- a/.github/workflows/publish-docker-image-for-zkevm.yml +++ b/.github/workflows/publish-docker-image-for-zkevm.yml @@ -15,7 +15,7 @@ jobs: name: Push Docker image to Docker Hub runs-on: ubuntu-latest env: - RELEASE_VERSION: 5.4.0 + RELEASE_VERSION: 6.0.0 DOCKER_CHAIN_NAME: zkevm steps: - name: Check out the repo diff --git a/.github/workflows/publish-docker-image-for-zksync.yml b/.github/workflows/publish-docker-image-for-zksync.yml index aa63176479..04cdf569a6 100644 --- a/.github/workflows/publish-docker-image-for-zksync.yml +++ b/.github/workflows/publish-docker-image-for-zksync.yml @@ -15,7 +15,7 @@ jobs: name: Push Docker image to Docker Hub runs-on: ubuntu-latest env: - RELEASE_VERSION: 5.4.0 + RELEASE_VERSION: 6.0.0 DOCKER_CHAIN_NAME: zksync steps: - name: Check out the repo diff --git a/.github/workflows/publish-docker-image-staging-on-demand.yml b/.github/workflows/publish-docker-image-staging-on-demand.yml index e33a3ee7c3..090a6be9d7 100644 --- a/.github/workflows/publish-docker-image-staging-on-demand.yml +++ b/.github/workflows/publish-docker-image-staging-on-demand.yml @@ -12,7 +12,7 @@ on: env: OTP_VERSION: '25.2.1' ELIXIR_VERSION: '1.14.5' - RELEASE_VERSION: 5.4.0 + RELEASE_VERSION: 6.0.0 jobs: push_to_registry: diff --git a/.github/workflows/release-additional.yml b/.github/workflows/release-additional.yml index 36ff36c1aa..f0912540dc 100644 --- a/.github/workflows/release-additional.yml +++ b/.github/workflows/release-additional.yml @@ -18,7 +18,7 @@ jobs: name: Push Docker image to Docker Hub runs-on: ubuntu-latest env: - RELEASE_VERSION: 5.4.0 + RELEASE_VERSION: 6.0.0 steps: - name: Check out the repo uses: actions/checkout@v4 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cd75990148..ded48ce497 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -18,7 +18,7 @@ jobs: name: Push Docker image to Docker Hub runs-on: ubuntu-latest env: - RELEASE_VERSION: 5.4.0 + RELEASE_VERSION: 6.0.0 steps: - name: Check out the repo uses: actions/checkout@v4 diff --git a/CHANGELOG.md b/CHANGELOG.md index c1df85bc68..5dc5c4caff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # ChangeLog +## Current + +### Features + +### Fixes + +### Chore + +
+ Dependencies version bumps + +
+ ## 6.0.0-dev ### Features @@ -65,6 +78,7 @@ - [#9094](https://github.com/blockscout/blockscout/pull/9094) - Improve exchange rates logging - [#9014](https://github.com/blockscout/blockscout/pull/9014) - Decrease amount of NFT in address collection: 15 -> 9 - [#8994](https://github.com/blockscout/blockscout/pull/8994) - Refactor transactions event preloads +- [#8991](https://github.com/blockscout/blockscout/pull/8991) - Manage DB queue target via runtime env var
Dependencies version bumps diff --git a/apps/block_scout_web/mix.exs b/apps/block_scout_web/mix.exs index d155e0c572..e90612b582 100644 --- a/apps/block_scout_web/mix.exs +++ b/apps/block_scout_web/mix.exs @@ -23,7 +23,7 @@ defmodule BlockScoutWeb.Mixfile do dialyzer: :test ], start_permanent: Mix.env() == :prod, - version: "5.4.0", + version: "6.0.0", xref: [exclude: [Explorer.Chain.Zkevm.Reader]] ] end diff --git a/apps/ethereum_jsonrpc/mix.exs b/apps/ethereum_jsonrpc/mix.exs index 1da7647be6..3545774c33 100644 --- a/apps/ethereum_jsonrpc/mix.exs +++ b/apps/ethereum_jsonrpc/mix.exs @@ -23,7 +23,7 @@ defmodule EthereumJsonrpc.MixProject do dialyzer: :test ], start_permanent: Mix.env() == :prod, - version: "5.4.0" + version: "6.0.0" ] end diff --git a/apps/explorer/mix.exs b/apps/explorer/mix.exs index a4195751ef..ae066dfdc7 100644 --- a/apps/explorer/mix.exs +++ b/apps/explorer/mix.exs @@ -24,7 +24,7 @@ defmodule Explorer.Mixfile do dialyzer: :test ], start_permanent: Mix.env() == :prod, - version: "5.4.0", + version: "6.0.0", xref: [exclude: [BlockScoutWeb.WebRouter.Helpers]] ] end diff --git a/apps/indexer/mix.exs b/apps/indexer/mix.exs index 58b20d24aa..7e349261cc 100644 --- a/apps/indexer/mix.exs +++ b/apps/indexer/mix.exs @@ -14,7 +14,7 @@ defmodule Indexer.MixProject do elixirc_paths: elixirc_paths(Mix.env()), lockfile: "../../mix.lock", start_permanent: Mix.env() == :prod, - version: "5.4.0" + version: "6.0.0" ] end diff --git a/docker-compose/docker-compose.yml b/docker-compose/docker-compose.yml index 2bbda2c942..b00502a5a1 100644 --- a/docker-compose/docker-compose.yml +++ b/docker-compose/docker-compose.yml @@ -34,7 +34,7 @@ services: CACHE_TOTAL_GAS_USAGE_COUNTER_ENABLED: "" CACHE_ADDRESS_WITH_BALANCES_UPDATE_INTERVAL: "" ADMIN_PANEL_ENABLED: "" - RELEASE_VERSION: 5.4.0 + RELEASE_VERSION: 6.0.0 links: - db:database environment: diff --git a/docker/Makefile b/docker/Makefile index 09e9239564..63bcd32de6 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -10,7 +10,7 @@ STATS_CONTAINER_NAME := stats STATS_DB_CONTAINER_NAME := stats-postgres PROXY_CONTAINER_NAME := proxy PG_CONTAINER_NAME := postgres -RELEASE_VERSION ?= '5.4.0' +RELEASE_VERSION ?= '6.0.0' TAG := $(RELEASE_VERSION)-commit-$(shell git log -1 --pretty=format:"%h") STABLE_TAG := $(RELEASE_VERSION) diff --git a/mix.exs b/mix.exs index f144b85ac3..06f570b70e 100644 --- a/mix.exs +++ b/mix.exs @@ -7,7 +7,7 @@ defmodule BlockScout.Mixfile do [ # app: :block_scout, # aliases: aliases(config_env()), - version: "5.4.0", + version: "6.0.0", apps_path: "apps", deps: deps(), dialyzer: dialyzer(), diff --git a/rel/config.exs b/rel/config.exs index aef4c2f2d1..b7c3f38132 100644 --- a/rel/config.exs +++ b/rel/config.exs @@ -71,7 +71,7 @@ end # will be used by default release :blockscout do - set version: "5.4.0-beta" + set version: "6.0.0-beta" set applications: [ :runtime_tools, block_scout_web: :permanent, From d4e03a8e7fc2399d90e51b54cee7c0b6af9e25dc Mon Sep 17 00:00:00 2001 From: Qwerty5Uiop Date: Fri, 5 Jan 2024 15:54:28 +0400 Subject: [PATCH 31/33] Fix migration_finished? logic --- CHANGELOG.md | 2 ++ apps/explorer/lib/explorer/etherscan.ex | 3 ++- apps/explorer/lib/explorer/migrator/filling_migration.ex | 2 +- .../test/explorer/chain/cache/gas_price_oracle_test.exs | 3 +++ 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5dc5c4caff..80a642db0c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,8 @@ ### Fixes +- [#9101](https://github.com/blockscout/blockscout/pull/9101) - Fix migration_finished? logic + ### Chore - [#9055](https://github.com/blockscout/blockscout/pull/9055) - Add ASC indices for logs, token transfers, transactions diff --git a/apps/explorer/lib/explorer/etherscan.ex b/apps/explorer/lib/explorer/etherscan.ex index dba8be73eb..023f74e9da 100644 --- a/apps/explorer/lib/explorer/etherscan.ex +++ b/apps/explorer/lib/explorer/etherscan.ex @@ -556,6 +556,7 @@ defmodule Explorer.Etherscan do from_address_hash: tt.from_address_hash, to_address_hash: tt.to_address_hash, amount: tt.amount, + amounts: tt.amounts, transaction_nonce: t.nonce, transaction_index: t.index, transaction_gas: t.gas, @@ -567,7 +568,7 @@ defmodule Explorer.Etherscan do block_number: b.number, block_timestamp: b.timestamp, confirmations: fragment("? - ?", ^block_height, t.block_number), - token_id: tt.token_id, + token_ids: tt.token_ids, token_name: tt.token_name, token_symbol: tt.token_symbol, token_decimals: tt.token_decimals, diff --git a/apps/explorer/lib/explorer/migrator/filling_migration.ex b/apps/explorer/lib/explorer/migrator/filling_migration.ex index 05823e9517..7ca5b26989 100644 --- a/apps/explorer/lib/explorer/migrator/filling_migration.ex +++ b/apps/explorer/lib/explorer/migrator/filling_migration.ex @@ -27,7 +27,7 @@ defmodule Explorer.Migrator.FillingMigration do end def migration_finished? do - not Repo.exists?(unprocessed_data_query()) + MigrationStatus.get_status(migration_name()) == "completed" end @impl true diff --git a/apps/explorer/test/explorer/chain/cache/gas_price_oracle_test.exs b/apps/explorer/test/explorer/chain/cache/gas_price_oracle_test.exs index 20472f79ee..a30d4070f6 100644 --- a/apps/explorer/test/explorer/chain/cache/gas_price_oracle_test.exs +++ b/apps/explorer/test/explorer/chain/cache/gas_price_oracle_test.exs @@ -351,6 +351,7 @@ defmodule Explorer.Chain.Cache.GasPriceOracleTest do status: :ok, block_hash: block1.hash, block_number: block1.number, + block_timestamp: block1.timestamp, cumulative_gas_used: 884_322, gas_used: 106_025, index: 0, @@ -366,6 +367,7 @@ defmodule Explorer.Chain.Cache.GasPriceOracleTest do status: :ok, block_hash: block2.hash, block_number: block2.number, + block_timestamp: block2.timestamp, cumulative_gas_used: 884_322, gas_used: 106_025, index: 0, @@ -381,6 +383,7 @@ defmodule Explorer.Chain.Cache.GasPriceOracleTest do status: :ok, block_hash: block2.hash, block_number: block2.number, + block_timestamp: block2.timestamp, cumulative_gas_used: 884_322, gas_used: 106_025, index: 1, From a0e5f11fca9ec9d5cc7143f6b38f38b9e11b77df Mon Sep 17 00:00:00 2001 From: Qwerty5Uiop Date: Mon, 8 Jan 2024 14:08:56 +0400 Subject: [PATCH 32/33] Update migrator's cache if migration is already finished --- CHANGELOG.md | 1 + apps/explorer/lib/explorer/migrator/filling_migration.ex | 1 + 2 files changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 80a642db0c..3ff34b5b40 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ ### Fixes +- [#9113](https://github.com/blockscout/blockscout/pull/9113) - Fix migrators cache updating - [#9101](https://github.com/blockscout/blockscout/pull/9101) - Fix migration_finished? logic ### Chore diff --git a/apps/explorer/lib/explorer/migrator/filling_migration.ex b/apps/explorer/lib/explorer/migrator/filling_migration.ex index 7ca5b26989..507dfcb6e5 100644 --- a/apps/explorer/lib/explorer/migrator/filling_migration.ex +++ b/apps/explorer/lib/explorer/migrator/filling_migration.ex @@ -34,6 +34,7 @@ defmodule Explorer.Migrator.FillingMigration do def init(_) do case MigrationStatus.get_status(migration_name()) do "completed" -> + update_cache() :ignore _ -> From 4bf97cd0f1ba5031f242737b2d9611e0faec201a Mon Sep 17 00:00:00 2001 From: Viktor Baranov Date: Mon, 8 Jan 2024 16:58:09 +0300 Subject: [PATCH 33/33] Update CHANGELOG --- CHANGELOG.md | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ff34b5b40..701e81828f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,14 +13,20 @@
-## 6.0.0-dev +## 6.0.0 ### Features +- [#9112](https://github.com/blockscout/blockscout/pull/9112) - Add specific url for eth_call +- [#9044](https://github.com/blockscout/blockscout/pull/9044) - Expand gas price oracle functionality + ### Fixes - [#9113](https://github.com/blockscout/blockscout/pull/9113) - Fix migrators cache updating - [#9101](https://github.com/blockscout/blockscout/pull/9101) - Fix migration_finished? logic +- [#9062](https://github.com/blockscout/blockscout/pull/9062) - Fix blockscout-ens integration +- [#9061](https://github.com/blockscout/blockscout/pull/9061) - Arbitrum allow tx receipt gasUsedForL1 field +- [#8812](https://github.com/blockscout/blockscout/pull/8812) - Update existing tokens type if got transfer with higher type priority ### Chore @@ -34,24 +40,21 @@ - [#8996](https://github.com/blockscout/blockscout/pull/8996) - Refine token transfers token ids index - [#5322](https://github.com/blockscout/blockscout/pull/5322) - DB denormalization: block consensus and timestamp in transaction table -## Current - -### Features - -- [#9112](https://github.com/blockscout/blockscout/pull/9112) - Add specific url for eth_call -- [#9044](https://github.com/blockscout/blockscout/pull/9044) - Expand gas price oracle functionality - -### Fixes - -- [#9062](https://github.com/blockscout/blockscout/pull/9062) - Fix blockscout-ens integration -- [#9061](https://github.com/blockscout/blockscout/pull/9061) - Arbitrum allow tx receipt gasUsedForL1 field -- [#8812](https://github.com/blockscout/blockscout/pull/8812) - Update existing tokens type if got transfer with higher type priority - -### Chore -
Dependencies version bumps +- [#9059](https://github.com/blockscout/blockscout/pull/9059) - Bump redux from 5.0.0 to 5.0.1 in /apps/block_scout_web/assets +- [#9057](https://github.com/blockscout/blockscout/pull/9057) - Bump benchee from 1.2.0 to 1.3.0 +- [#9060](https://github.com/blockscout/blockscout/pull/9060) - Bump @amplitude/analytics-browser from 2.3.7 to 2.3.8 in /apps/block_scout_web/assets +- [#9084](https://github.com/blockscout/blockscout/pull/9084) - Bump @babel/preset-env from 7.23.6 to 7.23.7 in /apps/block_scout_web/assets +- [#9083](https://github.com/blockscout/blockscout/pull/9083) - Bump @babel/core from 7.23.6 to 7.23.7 in /apps/block_scout_web/assets +- [#9086](https://github.com/blockscout/blockscout/pull/9086) - Bump core-js from 3.34.0 to 3.35.0 in /apps/block_scout_web/assets +- [#9081](https://github.com/blockscout/blockscout/pull/9081) - Bump sweetalert2 from 11.10.1 to 11.10.2 in /apps/block_scout_web/assets +- [#9085](https://github.com/blockscout/blockscout/pull/9085) - Bump moment from 2.29.4 to 2.30.1 in /apps/block_scout_web/assets +- [#9087](https://github.com/blockscout/blockscout/pull/9087) - Bump postcss-loader from 7.3.3 to 7.3.4 in /apps/block_scout_web/assets +- [#9082](https://github.com/blockscout/blockscout/pull/9082) - Bump sass-loader from 13.3.2 to 13.3.3 in /apps/block_scout_web/assets +- [#9088](https://github.com/blockscout/blockscout/pull/9088) - Bump sass from 1.69.5 to 1.69.6 in /apps/block_scout_web/assets +
## 5.4.0-beta