diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index a2a55dcbd5..efcd2af83d 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -899,12 +899,9 @@ defmodule Explorer.Chain do def list_top_addresses do query = from(a in Address, - left_join: t in Transaction, - on: a.hash == t.from_address_hash, where: a.fetched_coin_balance > ^0, - group_by: [a.hash, a.fetched_coin_balance], order_by: [desc: a.fetched_coin_balance, asc: a.hash], - select: {a, fragment("coalesce(1 + max(?), 0)", t.nonce)}, + select: {a, fragment("coalesce(1 + ?, 0)", a.nonce)}, limit: 250 ) diff --git a/apps/explorer/lib/explorer/chain/address.ex b/apps/explorer/lib/explorer/chain/address.ex index c5e2df3c9a..5777606d84 100644 --- a/apps/explorer/lib/explorer/chain/address.ex +++ b/apps/explorer/lib/explorer/chain/address.ex @@ -8,7 +8,7 @@ defmodule Explorer.Chain.Address do alias Ecto.Changeset alias Explorer.Chain.{Address, Block, Data, Hash, InternalTransaction, SmartContract, Token, Wei} - @optional_attrs ~w(contract_code fetched_coin_balance fetched_coin_balance_block_number)a + @optional_attrs ~w(contract_code fetched_coin_balance fetched_coin_balance_block_number nonce)a @required_attrs ~w(hash)a @allowed_attrs @optional_attrs ++ @required_attrs @@ -34,7 +34,8 @@ defmodule Explorer.Chain.Address do contract_code: Data.t() | nil, names: %Ecto.Association.NotLoaded{} | [Address.Name.t()], inserted_at: DateTime.t(), - updated_at: DateTime.t() + updated_at: DateTime.t(), + nonce: non_neg_integer() | nil } @primary_key {:hash, Hash.Address, autogenerate: false} @@ -42,6 +43,7 @@ defmodule Explorer.Chain.Address do field(:fetched_coin_balance, Wei) field(:fetched_coin_balance_block_number, :integer) field(:contract_code, Data) + field(:nonce, :integer) has_one(:smart_contract, SmartContract) has_one(:token, Token, foreign_key: :contract_address_hash) diff --git a/apps/explorer/lib/explorer/chain/import/addresses.ex b/apps/explorer/lib/explorer/chain/import/addresses.ex index ae6c92e887..fd0546ef3a 100644 --- a/apps/explorer/lib/explorer/chain/import/addresses.ex +++ b/apps/explorer/lib/explorer/chain/import/addresses.ex @@ -95,18 +95,10 @@ defmodule Explorer.Chain.Import.Addresses do # MAX on two columns fetched_coin_balance_block_number: fragment( - """ - CASE WHEN EXCLUDED.fetched_coin_balance_block_number IS NOT NULL AND - (? IS NULL OR - EXCLUDED.fetched_coin_balance_block_number >= ?) THEN - EXCLUDED.fetched_coin_balance_block_number - ELSE ? - END - """, - address.fetched_coin_balance_block_number, - address.fetched_coin_balance_block_number, + "GREATEST(EXCLUDED.fetched_coin_balance_block_number, ?)", address.fetched_coin_balance_block_number - ) + ), + nonce: fragment("GREATEST(EXCLUDED.nonce, ?)", address.nonce) ] ] ) diff --git a/apps/explorer/priv/repo/migrations/20181106152300_add_nonce_to_addresses.exs b/apps/explorer/priv/repo/migrations/20181106152300_add_nonce_to_addresses.exs new file mode 100644 index 0000000000..871c1daf57 --- /dev/null +++ b/apps/explorer/priv/repo/migrations/20181106152300_add_nonce_to_addresses.exs @@ -0,0 +1,26 @@ +defmodule Explorer.Repo.Migrations.AddNonceToAddresses do + @moduledoc """ + Use `priv/repo/migrations/scripts/20181126182700_migrate_address_nonce.sql` to migrate data. + + ```sh + mix ecto.migrate + psql -d $DATABASE -a -f priv/repo/migrations/scripts/20181126182700_migrate_address_nonce.sql + ``` + """ + + use Ecto.Migration + + def up do + # Add nonce + alter table(:addresses) do + add(:nonce, :integer) + end + end + + def down do + # Remove nonce + alter table(:addresses) do + remove(:nonce) + end + end +end diff --git a/apps/explorer/priv/repo/migrations/20181126203826_add_index_to_addresses.exs b/apps/explorer/priv/repo/migrations/20181126203826_add_index_to_addresses.exs new file mode 100644 index 0000000000..3c1b16dfec --- /dev/null +++ b/apps/explorer/priv/repo/migrations/20181126203826_add_index_to_addresses.exs @@ -0,0 +1,13 @@ +defmodule Explorer.Repo.Migrations.AddIndexToAddresses do + use Ecto.Migration + + def up do + execute( + "CREATE INDEX addresses_fetched_coin_balance_hash_index ON addresses (fetched_coin_balance DESC, hash ASC) WHERE fetched_coin_balance > 0" + ) + end + + def down do + execute("DROP INDEX addresses_fetched_coin_balance_hash_index") + end +end diff --git a/apps/explorer/priv/repo/migrations/scripts/20181126182700_migrate_address_nonce.sql b/apps/explorer/priv/repo/migrations/scripts/20181126182700_migrate_address_nonce.sql new file mode 100644 index 0000000000..bae18c3e84 --- /dev/null +++ b/apps/explorer/priv/repo/migrations/scripts/20181126182700_migrate_address_nonce.sql @@ -0,0 +1,55 @@ +DO $$ + DECLARE + row_count integer := 1; + batch_size integer := 50000; -- HOW MANY ITEMS WILL BE UPDATED AT TIME + iterator integer := batch_size; + affected integer; + BEGIN + DROP TABLE IF EXISTS addresses_nonce_temp; + + -- CREATES TEMP TABLE TO STORE THE ADDRESS NONCE TO BE UPDATED + CREATE TEMP TABLE addresses_nonce_temp( + from_address_hash bytea, + nonce integer, + row_number integer + ); + + INSERT INTO addresses_nonce_temp + SELECT DISTINCT ON (from_address_hash) + from_address_hash, + nonce, + ROW_NUMBER () OVER () + FROM transactions + ORDER BY from_address_hash, nonce DESC; + + row_count := (SELECT count(*) FROM addresses_nonce_temp); + + RAISE NOTICE '% items to be updated', row_count; + + -- ITERATES THROUGH THE ITEMS UNTIL THE TEMP TABLE IS EMPTY + WHILE row_count > 0 LOOP + -- UPDATES THE ADDRESS NONCE AND RETURNS THE ADDRESS_HASH + WITH updated_addresses AS ( + UPDATE addresses SET nonce = addresses_nonce_temp.nonce + FROM addresses_nonce_temp + WHERE addresses_nonce_temp.from_address_hash = addresses.hash + AND addresses_nonce_temp.row_number <= iterator + RETURNING addresses_nonce_temp.from_address_hash + ) + DELETE FROM addresses_nonce_temp + WHERE (from_address_hash) IN (select from_address_hash from updated_addresses); + + GET DIAGNOSTICS affected = ROW_COUNT; + RAISE NOTICE '-> % addresses updated!', affected; + + -- COMMITS THE BATCH UPDATES + CHECKPOINT; + + -- UPDATES THE COUNTER SO IT DOESN'T TURN INTO A INFINITE LOOP + row_count := (SELECT count(*) FROM addresses_nonce_temp); + iterator := iterator + batch_size; + + RAISE NOTICE '-> % counter', row_count; + RAISE NOTICE '-> % next batch', iterator; + END LOOP; + END $$; diff --git a/apps/indexer/lib/indexer/address_extraction.ex b/apps/indexer/lib/indexer/address_extraction.ex index 995465589a..9dbfb43cd6 100644 --- a/apps/indexer/lib/indexer/address_extraction.ex +++ b/apps/indexer/lib/indexer/address_extraction.ex @@ -84,7 +84,8 @@ defmodule Indexer.AddressExtraction do ], [ %{from: :block_number, to: :fetched_coin_balance_block_number}, - %{from: :from_address_hash, to: :hash} + %{from: :from_address_hash, to: :hash}, + %{from: :nonce, to: :nonce} ], [ %{from: :block_number, to: :fetched_coin_balance_block_number}, @@ -136,6 +137,7 @@ defmodule Indexer.AddressExtraction do required(:hash) => String.t(), required(:fetched_coin_balance_block_number) => non_neg_integer(), optional(:fetched_coin_balance) => non_neg_integer(), + optional(:nonce) => non_neg_integer(), optional(:contract_code) => String.t() } @@ -209,11 +211,13 @@ defmodule Indexer.AddressExtraction do ...> %{ ...> block_number: 1, ...> from_address_hash: "0x0000000000000000000000000000000000000001", - ...> to_address_hash: "0x0000000000000000000000000000000000000002" + ...> to_address_hash: "0x0000000000000000000000000000000000000002", + ...> nonce: 3 ...> }, ...> %{ ...> block_number: 2, - ...> from_address_hash: "0x0000000000000000000000000000000000000003" + ...> from_address_hash: "0x0000000000000000000000000000000000000003", + ...> nonce: 4 ...> } ...> ] ...> } @@ -221,7 +225,8 @@ defmodule Indexer.AddressExtraction do [ %{ fetched_coin_balance_block_number: 1, - hash: "0x0000000000000000000000000000000000000001" + hash: "0x0000000000000000000000000000000000000001", + nonce: 3 }, %{ fetched_coin_balance_block_number: 1, @@ -229,7 +234,8 @@ defmodule Indexer.AddressExtraction do }, %{ fetched_coin_balance_block_number: 2, - hash: "0x0000000000000000000000000000000000000003" + hash: "0x0000000000000000000000000000000000000003", + nonce: 4 } ] @@ -279,11 +285,13 @@ defmodule Indexer.AddressExtraction do ...> transactions: [ ...> %{ ...> block_number: 3, - ...> to_address_hash: "0x0000000000000000000000000000000000000001" + ...> to_address_hash: "0x0000000000000000000000000000000000000001", + ...> nonce: 5 ...> }, ...> %{ ...> block_number: 2, - ...> from_address_hash: "0x0000000000000000000000000000000000000001" + ...> from_address_hash: "0x0000000000000000000000000000000000000001", + ...> nonce: 4 ...> } ...> ], ...> logs: [ @@ -297,7 +305,8 @@ defmodule Indexer.AddressExtraction do [ %{ fetched_coin_balance_block_number: 7, - hash: "0x0000000000000000000000000000000000000001" + hash: "0x0000000000000000000000000000000000000001", + nonce: 4 } ] @@ -316,11 +325,13 @@ defmodule Indexer.AddressExtraction do ...> transactions: [ ...> %{ ...> block_number: 2, - ...> from_address_hash: "0x0000000000000000000000000000000000000001" + ...> from_address_hash: "0x0000000000000000000000000000000000000001", + ...> nonce: 4 ...> }, ...> %{ ...> block_number: 3, - ...> to_address_hash: "0x0000000000000000000000000000000000000001" + ...> to_address_hash: "0x0000000000000000000000000000000000000001", + ...> nonce: 5 ...> } ...> ] ...> } @@ -329,7 +340,8 @@ defmodule Indexer.AddressExtraction do %{ contract_code: "0x", fetched_coin_balance_block_number: 3, - hash: "0x0000000000000000000000000000000000000001" + hash: "0x0000000000000000000000000000000000000001", + nonce: 4 } ] @@ -366,6 +378,7 @@ defmodule Indexer.AddressExtraction do %{ required(:block_number) => non_neg_integer(), required(:from_address_hash) => String.t(), + required(:nonce) => non_neg_integer(), optional(:to_address_hash) => String.t(), optional(:created_contract_address_hash) => String.t() } @@ -445,38 +458,42 @@ defmodule Indexer.AddressExtraction do # Ensure that when `:addresses` or `:address_coin_balances` are present, their :fetched_coin_balance will win defp merge_addresses(%{hash: hash} = first, %{hash: hash} = second) do - case {first[:fetched_coin_balance], second[:fetched_coin_balance]} do - {nil, nil} -> - first - |> Map.merge(second) - |> Map.put( - :fetched_coin_balance_block_number, - max_nil_last( - Map.get(first, :fetched_coin_balance_block_number), - Map.get(second, :fetched_coin_balance_block_number) + merged_addresses = + case {first[:fetched_coin_balance], second[:fetched_coin_balance]} do + {nil, nil} -> + first + |> Map.merge(second) + |> Map.put( + :fetched_coin_balance_block_number, + max_nil_last( + Map.get(first, :fetched_coin_balance_block_number), + Map.get(second, :fetched_coin_balance_block_number) + ) ) - ) - - {nil, _} -> - # merge in `second` so its balance and block_number wins - Map.merge(first, second) - - {_, nil} -> - # merge in `first` so its balance and block_number wins - Map.merge(second, first) - - {_, _} -> - if greater_than_nil_last( - Map.get(first, :fetched_coin_balance_block_number), - Map.get(second, :fetched_coin_balance_block_number) - ) do - # merge in `first` so its block number wins - Map.merge(second, first) - else - # merge in `second` so its block number wins + + {nil, _} -> + # merge in `second` so its balance and block_number wins Map.merge(first, second) - end - end + + {_, nil} -> + # merge in `first` so its balance and block_number wins + Map.merge(second, first) + + {_, _} -> + if greater_than_nil_last( + Map.get(first, :fetched_coin_balance_block_number), + Map.get(second, :fetched_coin_balance_block_number) + ) do + # merge in `first` so its block number wins + Map.merge(second, first) + else + # merge in `second` so its block number wins + Map.merge(first, second) + end + end + + merged_addresses + |> Map.put(:nonce, max_nil_last(first[:nonce], second[:nonce])) end # `nil > 5 == true`, but we want numbers instead diff --git a/apps/indexer/test/indexer/address_extraction_test.exs b/apps/indexer/test/indexer/address_extraction_test.exs index 4b03173c99..8b5a8a4dfc 100644 --- a/apps/indexer/test/indexer/address_extraction_test.exs +++ b/apps/indexer/test/indexer/address_extraction_test.exs @@ -76,7 +76,8 @@ defmodule Indexer.AddressExtractionTest do %{ fetched_coin_balance_block_number: 2, contract_code: "0x1", - hash: "0x0000000000000000000000000000000000000001" + hash: "0x0000000000000000000000000000000000000001", + nonce: nil } ] end @@ -95,7 +96,8 @@ defmodule Indexer.AddressExtractionTest do transaction = %{ block_number: 3, from_address_hash: gen_hash(), - to_address_hash: gen_hash() + to_address_hash: gen_hash(), + nonce: 7 } log = %{address_hash: gen_hash(), block_number: 4} @@ -136,7 +138,11 @@ defmodule Indexer.AddressExtractionTest do contract_code: internal_transaction.created_contract_code, fetched_coin_balance_block_number: internal_transaction.block_number }, - %{hash: transaction.from_address_hash, fetched_coin_balance_block_number: transaction.block_number}, + %{ + hash: transaction.from_address_hash, + fetched_coin_balance_block_number: transaction.block_number, + nonce: 7 + }, %{hash: transaction.to_address_hash, fetched_coin_balance_block_number: transaction.block_number}, %{hash: log.address_hash, fetched_coin_balance_block_number: log.block_number}, %{ @@ -176,7 +182,7 @@ defmodule Indexer.AddressExtractionTest do blockchain_data = %{ blocks: [%{miner_hash: hash, number: 34}], - transactions: [%{block_number: 34, from_address_hash: hash}], + transactions: [%{block_number: 34, from_address_hash: hash, nonce: 12}], internal_transactions: [ %{ block_number: 34, @@ -188,7 +194,7 @@ defmodule Indexer.AddressExtractionTest do assert AddressExtraction.extract_addresses(blockchain_data) == [ - %{hash: hash, fetched_coin_balance_block_number: 34, contract_code: "code"} + %{hash: hash, fetched_coin_balance_block_number: 34, contract_code: "code", nonce: 12} ] end