Merge branch 'pp-pending-blocks-ops' into vb-speedup-address-to-logs-query

CHANGELOG.md

@@ -1,8 +1,10 @@
 ## Current
 
 ### Features
+- [#2835](https://github.com/poanetwork/blockscout/pull/2835), [#2871](https://github.com/poanetwork/blockscout/pull/2871), [#2872](https://github.com/poanetwork/blockscout/pull/2872), [#2886](https://github.com/poanetwork/blockscout/pull/2886), [#2925](https://github.com/poanetwork/blockscout/pull/2925) - Add "block_hash" to logs, token_transfers and internal transactions and "pending blocks operations" approach
 
 ### Fixes
+- [#2928](https://github.com/poanetwork/blockscout/pull/2928) - Speedup pending block ops int txs to fetch query
 - [#2924](https://github.com/poanetwork/blockscout/pull/2924) - Speedup address to logs query
 - [#2906](https://github.com/poanetwork/blockscout/pull/2906) - fix address sum cache
 - [#2902](https://github.com/poanetwork/blockscout/pull/2902) - Offset in blocks retrieval for average block time
@@ -33,8 +35,6 @@
 - [#2830](https://github.com/poanetwork/blockscout/pull/2830) - Fix wrong color of contract icon on xDai chain
 - [#2829](https://github.com/poanetwork/blockscout/pull/2829) - Fix for stuck gas limit label and value
 - [#2828](https://github.com/poanetwork/blockscout/pull/2828) - Fix for script that clears compilation/launching assets
-- [#2872](https://github.com/poanetwork/blockscout/pull/2872) - do not remove token transfers
-- [#2871](https://github.com/poanetwork/blockscout/pull/2871) - do not remove logs
 - [#2800](https://github.com/poanetwork/blockscout/pull/2800) - return not found for not verified contract for token read_contract
 - [#2806](https://github.com/poanetwork/blockscout/pull/2806) - Fix blocks fetching on the main page
 - [#2803](https://github.com/poanetwork/blockscout/pull/2803) - Fix block validator custom tooltip
@@ -50,7 +50,6 @@
 - [#2844](https://github.com/poanetwork/blockscout/pull/2844) - Extend external reward types up to 20
 - [#2827](https://github.com/poanetwork/blockscout/pull/2827) - Node js 12.13.0 (latest LTS release) support
 - [#2818](https://github.com/poanetwork/blockscout/pull/2818) - allow hiding marketcap percentage
-- [#2835](https://github.com/poanetwork/blockscout/pull/2835) - Update int txs constraints repairing script
 - [#2817](https://github.com/poanetwork/blockscout/pull/2817) - move docker integration documentation to blockscout docs
 - [#2808](https://github.com/poanetwork/blockscout/pull/2808) - Add tooltip for tx input
 - [#2807](https://github.com/poanetwork/blockscout/pull/2807) - 422 page

apps/explorer/priv/repo/migrations/20191018140054_add_pending_internal_txs_operation.exs

@@ -48,8 +48,54 @@ defmodule Explorer.Repo.Migrations.AddPendingInternalTxsOperation do
       ON t.hash = i.transaction_hash
     ) AS with_block
     WHERE itx.transaction_hash = with_block.transaction_hash
-    AND itx.index = with_block.index
-    ;
+    AND itx.index = with_block.index;
+    """)
+
+    execute("""
+    DELETE FROM internal_transactions WHERE block_hash IS NULL;
+    """)
+
+    execute("""
+    DO $$
+    DECLARE
+        duplicates_count INTEGER := 0;
+        blocks_scanned INTEGER := 0;
+        temprow RECORD;
+    BEGIN
+        FOR temprow IN
+            SELECT number, hash FROM blocks
+        LOOP
+            blocks_scanned := blocks_scanned + 1;
+            IF EXISTS (
+                SELECT 1 FROM transactions WHERE block_hash = temprow.hash
+            ) THEN
+                IF EXISTS (
+                    SELECT block_hash, block_index FROM internal_transactions
+                    WHERE block_hash = temprow.hash
+                    GROUP BY block_hash, block_index HAVING COUNT(*) > 1
+                ) THEN
+                    duplicates_count := duplicates_count + 1;
+                    RAISE NOTICE '% duplicates, blocks scanned %, block #%, block hash is %', duplicates_count, blocks_scanned, temprow.number , temprow.hash;
+
+                    IF NOT EXISTS (
+                        SELECT 1 FROM pending_block_operations
+                        WHERE block_hash = temprow.hash
+                    ) THEN
+                        INSERT INTO pending_block_operations
+                        (block_hash, inserted_at, updated_at, fetch_internal_transactions)
+                        SELECT b.hash, now(), now(), TRUE FROM blocks b
+                        WHERE b.hash = temprow.hash;
+                    END IF;
+
+                    DELETE FROM internal_transactions
+                    WHERE block_hash = temprow.hash;
+
+                    RAISE NOTICE 'DELETED';
+                END IF;
+            END IF;
+        END LOOP;
+        RAISE NOTICE 'SCRIPT FINISHED';
+    END $$;
     """)
 
     execute("""
@@ -70,5 +116,7 @@ defmodule Explorer.Repo.Migrations.AddPendingInternalTxsOperation do
         name: :internal_transactions_transaction_hash_index_index
       )
     )
+
+    create_if_not_exists(index(:internal_transactions, [:transaction_hash, :index]))
   end
 end

apps/explorer/priv/repo/migrations/20191121064805_add_block_hash_and_block_index_to_logs.exs

@@ -19,12 +19,9 @@ defmodule Explorer.Repo.Migrations.AddBlockHashAndBlockIndexToLogs do
       JOIN transactions t
       ON t.hash = l.transaction_hash
     ) AS with_block
-    WHERE log.transaction_hash = with_block.transaction_hash
-    ;
+    WHERE log.transaction_hash = with_block.transaction_hash;
     """)
 
-    create(index(:logs, [:block_number]))
-
     execute("""
     DELETE FROM logs WHERE block_hash IS NULL;
     """)

apps/explorer/priv/repo/migrations/20191122062035_add_block_hash_to_token_transfers.exs

@@ -16,8 +16,11 @@ defmodule Explorer.Repo.Migrations.AddBlockHashToTokenTransfers do
       JOIN transactions t
       ON t.hash = transfer.transaction_hash
     ) AS with_block
-    WHERE token_transfer.transaction_hash = with_block.transaction_hash
-    ;
+    WHERE token_transfer.transaction_hash = with_block.transaction_hash;
+    """)
+
+    execute("""
+    DELETE FROM token_transfers WHERE block_hash IS NULL;
     """)
 
     alter table(:token_transfers) do

apps/explorer/priv/repo/migrations/20191220113006_pending_block_operations_block_hash_partial_index.exs

@@ -0,0 +1,9 @@
+defmodule Explorer.Repo.Migrations.PendingBlockOperationsBlockHashPartialIndex do
+  use Ecto.Migration
+
+  def change do
+    execute(
+      "CREATE INDEX pending_block_operations_block_hash_index_partial ON pending_block_operations(block_hash) WHERE fetch_internal_transactions=true;"
+    )
+  end
+end