From bccf71c7484de35e42c2cf17eaaa51712ea46873 Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Wed, 18 Dec 2019 19:07:31 +0300 Subject: [PATCH 1/2] New indexing scheme update migration --- CHANGELOG.md | 6 +-- ...054_add_pending_internal_txs_operation.exs | 52 ++++++++++++++++++- ...add_block_hash_and_block_index_to_logs.exs | 5 +- ...2035_add_block_hash_to_token_transfers.exs | 7 ++- 4 files changed, 57 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ec51506d9c..c7f9f655fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,14 +1,13 @@ ## Current ### Features +- [#2835](https://github.com/poanetwork/blockscout/pull/2835), [#2871](https://github.com/poanetwork/blockscout/pull/2871), [#2872](https://github.com/poanetwork/blockscout/pull/2872), [#2886](https://github.com/poanetwork/blockscout/pull/2886), [#2925](https://github.com/poanetwork/blockscout/pull/2925) - Add "block_hash" to logs, token_transfers and internal transactions and "pending blocks operations" approach ### Fixes - [#2906](https://github.com/poanetwork/blockscout/pull/2906) - fix address sum cache - - [#2902](https://github.com/poanetwork/blockscout/pull/2902) - Offset in blocks retrieval for average block time ### Chore - - [#2896](https://github.com/poanetwork/blockscout/pull/2896) - Disable Parity websockets tests @@ -34,8 +33,6 @@ - [#2830](https://github.com/poanetwork/blockscout/pull/2830) - Fix wrong color of contract icon on xDai chain - [#2829](https://github.com/poanetwork/blockscout/pull/2829) - Fix for stuck gas limit label and value - [#2828](https://github.com/poanetwork/blockscout/pull/2828) - Fix for script that clears compilation/launching assets -- [#2872](https://github.com/poanetwork/blockscout/pull/2872) - do not remove token transfers -- [#2871](https://github.com/poanetwork/blockscout/pull/2871) - do not remove logs - [#2800](https://github.com/poanetwork/blockscout/pull/2800) - return not found for not verified contract for token read_contract - [#2806](https://github.com/poanetwork/blockscout/pull/2806) - Fix blocks fetching on the main page - [#2803](https://github.com/poanetwork/blockscout/pull/2803) - Fix block validator custom tooltip @@ -51,7 +48,6 @@ - [#2844](https://github.com/poanetwork/blockscout/pull/2844) - Extend external reward types up to 20 - [#2827](https://github.com/poanetwork/blockscout/pull/2827) - Node js 12.13.0 (latest LTS release) support - [#2818](https://github.com/poanetwork/blockscout/pull/2818) - allow hiding marketcap percentage -- [#2835](https://github.com/poanetwork/blockscout/pull/2835) - Update int txs constraints repairing script - [#2817](https://github.com/poanetwork/blockscout/pull/2817) - move docker integration documentation to blockscout docs - [#2808](https://github.com/poanetwork/blockscout/pull/2808) - Add tooltip for tx input - [#2807](https://github.com/poanetwork/blockscout/pull/2807) - 422 page diff --git a/apps/explorer/priv/repo/migrations/20191018140054_add_pending_internal_txs_operation.exs b/apps/explorer/priv/repo/migrations/20191018140054_add_pending_internal_txs_operation.exs index a12fa48836..d603d1e853 100644 --- a/apps/explorer/priv/repo/migrations/20191018140054_add_pending_internal_txs_operation.exs +++ b/apps/explorer/priv/repo/migrations/20191018140054_add_pending_internal_txs_operation.exs @@ -48,8 +48,54 @@ defmodule Explorer.Repo.Migrations.AddPendingInternalTxsOperation do ON t.hash = i.transaction_hash ) AS with_block WHERE itx.transaction_hash = with_block.transaction_hash - AND itx.index = with_block.index - ; + AND itx.index = with_block.index; + """) + + execute(""" + DELETE FROM internal_transactions WHERE block_hash IS NULL; + """) + + execute(""" + DO $$ + DECLARE + duplicates_count INTEGER := 0; + blocks_scanned INTEGER := 0; + temprow RECORD; + BEGIN + FOR temprow IN + SELECT number, hash FROM blocks + LOOP + blocks_scanned := blocks_scanned + 1; + IF EXISTS ( + SELECT 1 FROM transactions WHERE block_hash = temprow.hash + ) THEN + IF EXISTS ( + SELECT block_hash, block_index FROM internal_transactions + WHERE block_hash = temprow.hash + GROUP BY block_hash, block_index HAVING COUNT(*) > 1 + ) THEN + duplicates_count := duplicates_count + 1; + RAISE NOTICE '% duplicates, blocks scanned %, block #%, block hash is %', duplicates_count, blocks_scanned, temprow.number , temprow.hash; + + IF NOT EXISTS ( + SELECT 1 FROM pending_block_operations + WHERE block_hash = temprow.hash + ) THEN + INSERT INTO pending_block_operations + (block_hash, inserted_at, updated_at, fetch_internal_transactions) + SELECT b.hash, now(), now(), TRUE FROM blocks b + WHERE b.hash = temprow.hash; + END IF; + + DELETE FROM internal_transactions + WHERE block_hash = temprow.hash; + + RAISE NOTICE 'DELETED'; + END IF; + END IF; + END LOOP; + RAISE NOTICE 'SCRIPT FINISHED'; + END $$; """) execute(""" @@ -70,5 +116,7 @@ defmodule Explorer.Repo.Migrations.AddPendingInternalTxsOperation do name: :internal_transactions_transaction_hash_index_index ) ) + + create_if_not_exists(index(:internal_transactions, [:transaction_hash, :index])) end end diff --git a/apps/explorer/priv/repo/migrations/20191121064805_add_block_hash_and_block_index_to_logs.exs b/apps/explorer/priv/repo/migrations/20191121064805_add_block_hash_and_block_index_to_logs.exs index e873fb5951..94033d52d0 100644 --- a/apps/explorer/priv/repo/migrations/20191121064805_add_block_hash_and_block_index_to_logs.exs +++ b/apps/explorer/priv/repo/migrations/20191121064805_add_block_hash_and_block_index_to_logs.exs @@ -19,12 +19,9 @@ defmodule Explorer.Repo.Migrations.AddBlockHashAndBlockIndexToLogs do JOIN transactions t ON t.hash = l.transaction_hash ) AS with_block - WHERE log.transaction_hash = with_block.transaction_hash - ; + WHERE log.transaction_hash = with_block.transaction_hash; """) - create(index(:logs, [:block_number])) - execute(""" DELETE FROM logs WHERE block_hash IS NULL; """) diff --git a/apps/explorer/priv/repo/migrations/20191122062035_add_block_hash_to_token_transfers.exs b/apps/explorer/priv/repo/migrations/20191122062035_add_block_hash_to_token_transfers.exs index 5635cd9bd9..c10d3b2713 100644 --- a/apps/explorer/priv/repo/migrations/20191122062035_add_block_hash_to_token_transfers.exs +++ b/apps/explorer/priv/repo/migrations/20191122062035_add_block_hash_to_token_transfers.exs @@ -16,8 +16,11 @@ defmodule Explorer.Repo.Migrations.AddBlockHashToTokenTransfers do JOIN transactions t ON t.hash = transfer.transaction_hash ) AS with_block - WHERE token_transfer.transaction_hash = with_block.transaction_hash - ; + WHERE token_transfer.transaction_hash = with_block.transaction_hash; + """) + + execute(""" + DELETE FROM token_transfers WHERE block_hash IS NULL; """) alter table(:token_transfers) do From 55d280c05ca535c61dc1df59db2c8d4ef4533dd8 Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Fri, 20 Dec 2019 14:34:01 +0300 Subject: [PATCH 2/2] Partial index on pending_block_operations table on block hash where fetch_internal_transactions --- CHANGELOG.md | 3 +-- ...pending_block_operations_block_hash_partial_index.exs | 9 +++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 apps/explorer/priv/repo/migrations/20191220113006_pending_block_operations_block_hash_partial_index.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index ec51506d9c..811ca59583 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,12 +3,11 @@ ### Features ### Fixes +- [#2928](https://github.com/poanetwork/blockscout/pull/2928) - Speedup pending block ops int txs to fetch query - [#2906](https://github.com/poanetwork/blockscout/pull/2906) - fix address sum cache - - [#2902](https://github.com/poanetwork/blockscout/pull/2902) - Offset in blocks retrieval for average block time ### Chore - - [#2896](https://github.com/poanetwork/blockscout/pull/2896) - Disable Parity websockets tests diff --git a/apps/explorer/priv/repo/migrations/20191220113006_pending_block_operations_block_hash_partial_index.exs b/apps/explorer/priv/repo/migrations/20191220113006_pending_block_operations_block_hash_partial_index.exs new file mode 100644 index 0000000000..1b1a52e4ed --- /dev/null +++ b/apps/explorer/priv/repo/migrations/20191220113006_pending_block_operations_block_hash_partial_index.exs @@ -0,0 +1,9 @@ +defmodule Explorer.Repo.Migrations.PendingBlockOperationsBlockHashPartialIndex do + use Ecto.Migration + + def change do + execute( + "CREATE INDEX pending_block_operations_block_hash_index_partial ON pending_block_operations(block_hash) WHERE fetch_internal_transactions=true;" + ) + end +end