From 29199eb54e047c3e3c6b2e6c4eba9f9b93528e3b Mon Sep 17 00:00:00 2001 From: svenski123 Date: Tue, 6 Nov 2018 17:50:34 +0000 Subject: [PATCH 1/5] Add nonce field to Address object and addresses table. This caches the highest nonce value seen for sent transactions for a given address. The top addresses query no longer needs to scan the transactions table for nonce values. The database migration script adds the nonce field to the addresses table and sets the nonce value according to the contents of the transactions table. --- apps/explorer/lib/explorer/chain.ex | 5 +--- apps/explorer/lib/explorer/chain/address.ex | 6 ++-- .../lib/explorer/chain/import/addresses.ex | 14 ++------- .../201811061523_add_nonce_to_addresses.exs | 30 +++++++++++++++++++ .../indexer/lib/indexer/address_extraction.ex | 18 +++++++---- 5 files changed, 51 insertions(+), 22 deletions(-) create mode 100644 apps/explorer/priv/repo/migrations/201811061523_add_nonce_to_addresses.exs diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index a2a55dcbd5..efcd2af83d 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -899,12 +899,9 @@ defmodule Explorer.Chain do def list_top_addresses do query = from(a in Address, - left_join: t in Transaction, - on: a.hash == t.from_address_hash, where: a.fetched_coin_balance > ^0, - group_by: [a.hash, a.fetched_coin_balance], order_by: [desc: a.fetched_coin_balance, asc: a.hash], - select: {a, fragment("coalesce(1 + max(?), 0)", t.nonce)}, + select: {a, fragment("coalesce(1 + ?, 0)", a.nonce)}, limit: 250 ) diff --git a/apps/explorer/lib/explorer/chain/address.ex b/apps/explorer/lib/explorer/chain/address.ex index c5e2df3c9a..5777606d84 100644 --- a/apps/explorer/lib/explorer/chain/address.ex +++ b/apps/explorer/lib/explorer/chain/address.ex @@ -8,7 +8,7 @@ defmodule Explorer.Chain.Address do alias Ecto.Changeset alias Explorer.Chain.{Address, Block, Data, Hash, InternalTransaction, SmartContract, Token, Wei} - @optional_attrs ~w(contract_code fetched_coin_balance fetched_coin_balance_block_number)a + @optional_attrs ~w(contract_code fetched_coin_balance fetched_coin_balance_block_number nonce)a @required_attrs ~w(hash)a @allowed_attrs @optional_attrs ++ @required_attrs @@ -34,7 +34,8 @@ defmodule Explorer.Chain.Address do contract_code: Data.t() | nil, names: %Ecto.Association.NotLoaded{} | [Address.Name.t()], inserted_at: DateTime.t(), - updated_at: DateTime.t() + updated_at: DateTime.t(), + nonce: non_neg_integer() | nil } @primary_key {:hash, Hash.Address, autogenerate: false} @@ -42,6 +43,7 @@ defmodule Explorer.Chain.Address do field(:fetched_coin_balance, Wei) field(:fetched_coin_balance_block_number, :integer) field(:contract_code, Data) + field(:nonce, :integer) has_one(:smart_contract, SmartContract) has_one(:token, Token, foreign_key: :contract_address_hash) diff --git a/apps/explorer/lib/explorer/chain/import/addresses.ex b/apps/explorer/lib/explorer/chain/import/addresses.ex index ae6c92e887..fd0546ef3a 100644 --- a/apps/explorer/lib/explorer/chain/import/addresses.ex +++ b/apps/explorer/lib/explorer/chain/import/addresses.ex @@ -95,18 +95,10 @@ defmodule Explorer.Chain.Import.Addresses do # MAX on two columns fetched_coin_balance_block_number: fragment( - """ - CASE WHEN EXCLUDED.fetched_coin_balance_block_number IS NOT NULL AND - (? IS NULL OR - EXCLUDED.fetched_coin_balance_block_number >= ?) THEN - EXCLUDED.fetched_coin_balance_block_number - ELSE ? - END - """, - address.fetched_coin_balance_block_number, - address.fetched_coin_balance_block_number, + "GREATEST(EXCLUDED.fetched_coin_balance_block_number, ?)", address.fetched_coin_balance_block_number - ) + ), + nonce: fragment("GREATEST(EXCLUDED.nonce, ?)", address.nonce) ] ] ) diff --git a/apps/explorer/priv/repo/migrations/201811061523_add_nonce_to_addresses.exs b/apps/explorer/priv/repo/migrations/201811061523_add_nonce_to_addresses.exs new file mode 100644 index 0000000000..cbdfe4b1b9 --- /dev/null +++ b/apps/explorer/priv/repo/migrations/201811061523_add_nonce_to_addresses.exs @@ -0,0 +1,30 @@ +defmodule Explorer.Repo.Migrations.AddNonceToAddresses do + use Ecto.Migration + + def up do + # Add nonce + alter table(:addresses) do + add(:nonce, :integer) + end + + # Populate nonce field from transactions table + execute(""" + WITH t AS ( + SELECT from_address_hash AS hash, MAX(nonce) AS nonce + FROM transactions + GROUP BY hash + ) + UPDATE addresses AS a + SET nonce = t.nonce + FROM t + WHERE a.hash = t.hash + """) + end + + def down do + # Remove nonce + alter table(:addresses) do + remove(:nonce) + end + end +end diff --git a/apps/indexer/lib/indexer/address_extraction.ex b/apps/indexer/lib/indexer/address_extraction.ex index 995465589a..37e93e3987 100644 --- a/apps/indexer/lib/indexer/address_extraction.ex +++ b/apps/indexer/lib/indexer/address_extraction.ex @@ -84,7 +84,8 @@ defmodule Indexer.AddressExtraction do ], [ %{from: :block_number, to: :fetched_coin_balance_block_number}, - %{from: :from_address_hash, to: :hash} + %{from: :from_address_hash, to: :hash}, + %{from: :nonce, to: :nonce} ], [ %{from: :block_number, to: :fetched_coin_balance_block_number}, @@ -136,6 +137,7 @@ defmodule Indexer.AddressExtraction do required(:hash) => String.t(), required(:fetched_coin_balance_block_number) => non_neg_integer(), optional(:fetched_coin_balance) => non_neg_integer(), + optional(:nonce) => non_neg_integer(), optional(:contract_code) => String.t() } @@ -209,11 +211,13 @@ defmodule Indexer.AddressExtraction do ...> %{ ...> block_number: 1, ...> from_address_hash: "0x0000000000000000000000000000000000000001", - ...> to_address_hash: "0x0000000000000000000000000000000000000002" + ...> to_address_hash: "0x0000000000000000000000000000000000000002", + ...> nonce: 3 ...> }, ...> %{ ...> block_number: 2, - ...> from_address_hash: "0x0000000000000000000000000000000000000003" + ...> from_address_hash: "0x0000000000000000000000000000000000000003", + ...> nonce: 4 ...> } ...> ] ...> } @@ -221,7 +225,8 @@ defmodule Indexer.AddressExtraction do [ %{ fetched_coin_balance_block_number: 1, - hash: "0x0000000000000000000000000000000000000001" + hash: "0x0000000000000000000000000000000000000001", + nonce: 3 }, %{ fetched_coin_balance_block_number: 1, @@ -229,7 +234,8 @@ defmodule Indexer.AddressExtraction do }, %{ fetched_coin_balance_block_number: 2, - hash: "0x0000000000000000000000000000000000000003" + hash: "0x0000000000000000000000000000000000000003", + nonce: 4 } ] @@ -366,6 +372,7 @@ defmodule Indexer.AddressExtraction do %{ required(:block_number) => non_neg_integer(), required(:from_address_hash) => String.t(), + required(:nonce) => non_neg_integer(), optional(:to_address_hash) => String.t(), optional(:created_contract_address_hash) => String.t() } @@ -477,6 +484,7 @@ defmodule Indexer.AddressExtraction do Map.merge(first, second) end end + |> Map.put(:nonce, max_nil_last(first[:nonce], second[:nonce])) end # `nil > 5 == true`, but we want numbers instead From 3b45474e36b643d73dcf6ab6f6715f274ed527af Mon Sep 17 00:00:00 2001 From: svenski123 Date: Tue, 6 Nov 2018 19:13:47 +0000 Subject: [PATCH 2/5] Update address extraction testcases --- .../indexer/lib/indexer/address_extraction.ex | 81 ++++++++++--------- .../test/indexer/address_extraction_test.exs | 16 ++-- 2 files changed, 56 insertions(+), 41 deletions(-) diff --git a/apps/indexer/lib/indexer/address_extraction.ex b/apps/indexer/lib/indexer/address_extraction.ex index 37e93e3987..9dbfb43cd6 100644 --- a/apps/indexer/lib/indexer/address_extraction.ex +++ b/apps/indexer/lib/indexer/address_extraction.ex @@ -285,11 +285,13 @@ defmodule Indexer.AddressExtraction do ...> transactions: [ ...> %{ ...> block_number: 3, - ...> to_address_hash: "0x0000000000000000000000000000000000000001" + ...> to_address_hash: "0x0000000000000000000000000000000000000001", + ...> nonce: 5 ...> }, ...> %{ ...> block_number: 2, - ...> from_address_hash: "0x0000000000000000000000000000000000000001" + ...> from_address_hash: "0x0000000000000000000000000000000000000001", + ...> nonce: 4 ...> } ...> ], ...> logs: [ @@ -303,7 +305,8 @@ defmodule Indexer.AddressExtraction do [ %{ fetched_coin_balance_block_number: 7, - hash: "0x0000000000000000000000000000000000000001" + hash: "0x0000000000000000000000000000000000000001", + nonce: 4 } ] @@ -322,11 +325,13 @@ defmodule Indexer.AddressExtraction do ...> transactions: [ ...> %{ ...> block_number: 2, - ...> from_address_hash: "0x0000000000000000000000000000000000000001" + ...> from_address_hash: "0x0000000000000000000000000000000000000001", + ...> nonce: 4 ...> }, ...> %{ ...> block_number: 3, - ...> to_address_hash: "0x0000000000000000000000000000000000000001" + ...> to_address_hash: "0x0000000000000000000000000000000000000001", + ...> nonce: 5 ...> } ...> ] ...> } @@ -335,7 +340,8 @@ defmodule Indexer.AddressExtraction do %{ contract_code: "0x", fetched_coin_balance_block_number: 3, - hash: "0x0000000000000000000000000000000000000001" + hash: "0x0000000000000000000000000000000000000001", + nonce: 4 } ] @@ -452,38 +458,41 @@ defmodule Indexer.AddressExtraction do # Ensure that when `:addresses` or `:address_coin_balances` are present, their :fetched_coin_balance will win defp merge_addresses(%{hash: hash} = first, %{hash: hash} = second) do - case {first[:fetched_coin_balance], second[:fetched_coin_balance]} do - {nil, nil} -> - first - |> Map.merge(second) - |> Map.put( - :fetched_coin_balance_block_number, - max_nil_last( - Map.get(first, :fetched_coin_balance_block_number), - Map.get(second, :fetched_coin_balance_block_number) + merged_addresses = + case {first[:fetched_coin_balance], second[:fetched_coin_balance]} do + {nil, nil} -> + first + |> Map.merge(second) + |> Map.put( + :fetched_coin_balance_block_number, + max_nil_last( + Map.get(first, :fetched_coin_balance_block_number), + Map.get(second, :fetched_coin_balance_block_number) + ) ) - ) - - {nil, _} -> - # merge in `second` so its balance and block_number wins - Map.merge(first, second) - - {_, nil} -> - # merge in `first` so its balance and block_number wins - Map.merge(second, first) - - {_, _} -> - if greater_than_nil_last( - Map.get(first, :fetched_coin_balance_block_number), - Map.get(second, :fetched_coin_balance_block_number) - ) do - # merge in `first` so its block number wins - Map.merge(second, first) - else - # merge in `second` so its block number wins + + {nil, _} -> + # merge in `second` so its balance and block_number wins Map.merge(first, second) - end - end + + {_, nil} -> + # merge in `first` so its balance and block_number wins + Map.merge(second, first) + + {_, _} -> + if greater_than_nil_last( + Map.get(first, :fetched_coin_balance_block_number), + Map.get(second, :fetched_coin_balance_block_number) + ) do + # merge in `first` so its block number wins + Map.merge(second, first) + else + # merge in `second` so its block number wins + Map.merge(first, second) + end + end + + merged_addresses |> Map.put(:nonce, max_nil_last(first[:nonce], second[:nonce])) end diff --git a/apps/indexer/test/indexer/address_extraction_test.exs b/apps/indexer/test/indexer/address_extraction_test.exs index 4b03173c99..8b5a8a4dfc 100644 --- a/apps/indexer/test/indexer/address_extraction_test.exs +++ b/apps/indexer/test/indexer/address_extraction_test.exs @@ -76,7 +76,8 @@ defmodule Indexer.AddressExtractionTest do %{ fetched_coin_balance_block_number: 2, contract_code: "0x1", - hash: "0x0000000000000000000000000000000000000001" + hash: "0x0000000000000000000000000000000000000001", + nonce: nil } ] end @@ -95,7 +96,8 @@ defmodule Indexer.AddressExtractionTest do transaction = %{ block_number: 3, from_address_hash: gen_hash(), - to_address_hash: gen_hash() + to_address_hash: gen_hash(), + nonce: 7 } log = %{address_hash: gen_hash(), block_number: 4} @@ -136,7 +138,11 @@ defmodule Indexer.AddressExtractionTest do contract_code: internal_transaction.created_contract_code, fetched_coin_balance_block_number: internal_transaction.block_number }, - %{hash: transaction.from_address_hash, fetched_coin_balance_block_number: transaction.block_number}, + %{ + hash: transaction.from_address_hash, + fetched_coin_balance_block_number: transaction.block_number, + nonce: 7 + }, %{hash: transaction.to_address_hash, fetched_coin_balance_block_number: transaction.block_number}, %{hash: log.address_hash, fetched_coin_balance_block_number: log.block_number}, %{ @@ -176,7 +182,7 @@ defmodule Indexer.AddressExtractionTest do blockchain_data = %{ blocks: [%{miner_hash: hash, number: 34}], - transactions: [%{block_number: 34, from_address_hash: hash}], + transactions: [%{block_number: 34, from_address_hash: hash, nonce: 12}], internal_transactions: [ %{ block_number: 34, @@ -188,7 +194,7 @@ defmodule Indexer.AddressExtractionTest do assert AddressExtraction.extract_addresses(blockchain_data) == [ - %{hash: hash, fetched_coin_balance_block_number: 34, contract_code: "code"} + %{hash: hash, fetched_coin_balance_block_number: 34, contract_code: "code", nonce: 12} ] end From a3364385c69315ea06f8380ca0dca17747009e61 Mon Sep 17 00:00:00 2001 From: svenski123 Date: Wed, 7 Nov 2018 23:31:47 +0000 Subject: [PATCH 3/5] Comment out update query in migration script due to run length concerns on a large existing database. --- .../20181106152300_add_nonce_to_addresses.exs | 31 +++++++++++++++++++ .../201811061523_add_nonce_to_addresses.exs | 30 ------------------ 2 files changed, 31 insertions(+), 30 deletions(-) create mode 100644 apps/explorer/priv/repo/migrations/20181106152300_add_nonce_to_addresses.exs delete mode 100644 apps/explorer/priv/repo/migrations/201811061523_add_nonce_to_addresses.exs diff --git a/apps/explorer/priv/repo/migrations/20181106152300_add_nonce_to_addresses.exs b/apps/explorer/priv/repo/migrations/20181106152300_add_nonce_to_addresses.exs new file mode 100644 index 0000000000..b6fd4f1594 --- /dev/null +++ b/apps/explorer/priv/repo/migrations/20181106152300_add_nonce_to_addresses.exs @@ -0,0 +1,31 @@ +defmodule Explorer.Repo.Migrations.AddNonceToAddresses do + use Ecto.Migration + + def up do + # Add nonce + alter table(:addresses) do + add(:nonce, :integer) + end + + # Populate nonce field from transactions table + # Commented out due to running time concerns + # execute(""" + # WITH t AS ( + # SELECT from_address_hash AS hash, MAX(nonce) AS nonce + # FROM transactions + # GROUP BY hash + # ) + # UPDATE addresses AS a + # SET nonce = t.nonce + # FROM t + # WHERE a.hash = t.hash + # """) + end + + def down do + # Remove nonce + alter table(:addresses) do + remove(:nonce) + end + end +end diff --git a/apps/explorer/priv/repo/migrations/201811061523_add_nonce_to_addresses.exs b/apps/explorer/priv/repo/migrations/201811061523_add_nonce_to_addresses.exs deleted file mode 100644 index cbdfe4b1b9..0000000000 --- a/apps/explorer/priv/repo/migrations/201811061523_add_nonce_to_addresses.exs +++ /dev/null @@ -1,30 +0,0 @@ -defmodule Explorer.Repo.Migrations.AddNonceToAddresses do - use Ecto.Migration - - def up do - # Add nonce - alter table(:addresses) do - add(:nonce, :integer) - end - - # Populate nonce field from transactions table - execute(""" - WITH t AS ( - SELECT from_address_hash AS hash, MAX(nonce) AS nonce - FROM transactions - GROUP BY hash - ) - UPDATE addresses AS a - SET nonce = t.nonce - FROM t - WHERE a.hash = t.hash - """) - end - - def down do - # Remove nonce - alter table(:addresses) do - remove(:nonce) - end - end -end From 40a9da30228b94b97e10d887d7fcb3b2f7b42169 Mon Sep 17 00:00:00 2001 From: Amanda Sposito Date: Mon, 26 Nov 2018 18:33:22 -0200 Subject: [PATCH 4/5] Add script to migrate nonce to addresses This script copy nonce from the transactions' table to addresses' table in batches. --- .../20181106152300_add_nonce_to_addresses.exs | 23 +++----- .../20181126182700_migrate_address_nonce.sql | 55 +++++++++++++++++++ 2 files changed, 64 insertions(+), 14 deletions(-) create mode 100644 apps/explorer/priv/repo/migrations/scripts/20181126182700_migrate_address_nonce.sql diff --git a/apps/explorer/priv/repo/migrations/20181106152300_add_nonce_to_addresses.exs b/apps/explorer/priv/repo/migrations/20181106152300_add_nonce_to_addresses.exs index b6fd4f1594..871c1daf57 100644 --- a/apps/explorer/priv/repo/migrations/20181106152300_add_nonce_to_addresses.exs +++ b/apps/explorer/priv/repo/migrations/20181106152300_add_nonce_to_addresses.exs @@ -1,4 +1,13 @@ defmodule Explorer.Repo.Migrations.AddNonceToAddresses do + @moduledoc """ + Use `priv/repo/migrations/scripts/20181126182700_migrate_address_nonce.sql` to migrate data. + + ```sh + mix ecto.migrate + psql -d $DATABASE -a -f priv/repo/migrations/scripts/20181126182700_migrate_address_nonce.sql + ``` + """ + use Ecto.Migration def up do @@ -6,20 +15,6 @@ defmodule Explorer.Repo.Migrations.AddNonceToAddresses do alter table(:addresses) do add(:nonce, :integer) end - - # Populate nonce field from transactions table - # Commented out due to running time concerns - # execute(""" - # WITH t AS ( - # SELECT from_address_hash AS hash, MAX(nonce) AS nonce - # FROM transactions - # GROUP BY hash - # ) - # UPDATE addresses AS a - # SET nonce = t.nonce - # FROM t - # WHERE a.hash = t.hash - # """) end def down do diff --git a/apps/explorer/priv/repo/migrations/scripts/20181126182700_migrate_address_nonce.sql b/apps/explorer/priv/repo/migrations/scripts/20181126182700_migrate_address_nonce.sql new file mode 100644 index 0000000000..bae18c3e84 --- /dev/null +++ b/apps/explorer/priv/repo/migrations/scripts/20181126182700_migrate_address_nonce.sql @@ -0,0 +1,55 @@ +DO $$ + DECLARE + row_count integer := 1; + batch_size integer := 50000; -- HOW MANY ITEMS WILL BE UPDATED AT TIME + iterator integer := batch_size; + affected integer; + BEGIN + DROP TABLE IF EXISTS addresses_nonce_temp; + + -- CREATES TEMP TABLE TO STORE THE ADDRESS NONCE TO BE UPDATED + CREATE TEMP TABLE addresses_nonce_temp( + from_address_hash bytea, + nonce integer, + row_number integer + ); + + INSERT INTO addresses_nonce_temp + SELECT DISTINCT ON (from_address_hash) + from_address_hash, + nonce, + ROW_NUMBER () OVER () + FROM transactions + ORDER BY from_address_hash, nonce DESC; + + row_count := (SELECT count(*) FROM addresses_nonce_temp); + + RAISE NOTICE '% items to be updated', row_count; + + -- ITERATES THROUGH THE ITEMS UNTIL THE TEMP TABLE IS EMPTY + WHILE row_count > 0 LOOP + -- UPDATES THE ADDRESS NONCE AND RETURNS THE ADDRESS_HASH + WITH updated_addresses AS ( + UPDATE addresses SET nonce = addresses_nonce_temp.nonce + FROM addresses_nonce_temp + WHERE addresses_nonce_temp.from_address_hash = addresses.hash + AND addresses_nonce_temp.row_number <= iterator + RETURNING addresses_nonce_temp.from_address_hash + ) + DELETE FROM addresses_nonce_temp + WHERE (from_address_hash) IN (select from_address_hash from updated_addresses); + + GET DIAGNOSTICS affected = ROW_COUNT; + RAISE NOTICE '-> % addresses updated!', affected; + + -- COMMITS THE BATCH UPDATES + CHECKPOINT; + + -- UPDATES THE COUNTER SO IT DOESN'T TURN INTO A INFINITE LOOP + row_count := (SELECT count(*) FROM addresses_nonce_temp); + iterator := iterator + batch_size; + + RAISE NOTICE '-> % counter', row_count; + RAISE NOTICE '-> % next batch', iterator; + END LOOP; + END $$; From 8ad5a3103ec9337e3a1095ae4748611069b68483 Mon Sep 17 00:00:00 2001 From: Amanda Sposito Date: Tue, 27 Nov 2018 11:12:33 -0200 Subject: [PATCH 5/5] Add index to addresses table Creates a partial index using the fetched_coin_balance and hash columns --- .../20181126203826_add_index_to_addresses.exs | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 apps/explorer/priv/repo/migrations/20181126203826_add_index_to_addresses.exs diff --git a/apps/explorer/priv/repo/migrations/20181126203826_add_index_to_addresses.exs b/apps/explorer/priv/repo/migrations/20181126203826_add_index_to_addresses.exs new file mode 100644 index 0000000000..3c1b16dfec --- /dev/null +++ b/apps/explorer/priv/repo/migrations/20181126203826_add_index_to_addresses.exs @@ -0,0 +1,13 @@ +defmodule Explorer.Repo.Migrations.AddIndexToAddresses do + use Ecto.Migration + + def up do + execute( + "CREATE INDEX addresses_fetched_coin_balance_hash_index ON addresses (fetched_coin_balance DESC, hash ASC) WHERE fetched_coin_balance > 0" + ) + end + + def down do + execute("DROP INDEX addresses_fetched_coin_balance_hash_index") + end +end