Merge pull request #1140 from poanetwork/ams-top-accounts

Add nonce field to addresses table

apps/explorer/lib/explorer/chain.ex

@@ -899,12 +899,9 @@ defmodule Explorer.Chain do
   def list_top_addresses do
     query =
       from(a in Address,
-        left_join: t in Transaction,
-        on: a.hash == t.from_address_hash,
         where: a.fetched_coin_balance > ^0,
-        group_by: [a.hash, a.fetched_coin_balance],
         order_by: [desc: a.fetched_coin_balance, asc: a.hash],
-        select: {a, fragment("coalesce(1 + max(?), 0)", t.nonce)},
+        select: {a, fragment("coalesce(1 + ?, 0)", a.nonce)},
         limit: 250
       )
 

apps/explorer/lib/explorer/chain/address.ex

@@ -8,7 +8,7 @@ defmodule Explorer.Chain.Address do
   alias Ecto.Changeset
   alias Explorer.Chain.{Address, Block, Data, Hash, InternalTransaction, SmartContract, Token, Wei}
 
-  @optional_attrs ~w(contract_code fetched_coin_balance fetched_coin_balance_block_number)a
+  @optional_attrs ~w(contract_code fetched_coin_balance fetched_coin_balance_block_number nonce)a
   @required_attrs ~w(hash)a
   @allowed_attrs @optional_attrs ++ @required_attrs
 
@@ -34,7 +34,8 @@ defmodule Explorer.Chain.Address do
           contract_code: Data.t() | nil,
           names: %Ecto.Association.NotLoaded{} | [Address.Name.t()],
           inserted_at: DateTime.t(),
-          updated_at: DateTime.t()
+          updated_at: DateTime.t(),
+          nonce: non_neg_integer() | nil
         }
 
   @primary_key {:hash, Hash.Address, autogenerate: false}
@@ -42,6 +43,7 @@ defmodule Explorer.Chain.Address do
     field(:fetched_coin_balance, Wei)
     field(:fetched_coin_balance_block_number, :integer)
     field(:contract_code, Data)
+    field(:nonce, :integer)
 
     has_one(:smart_contract, SmartContract)
     has_one(:token, Token, foreign_key: :contract_address_hash)

apps/explorer/lib/explorer/chain/import/addresses.ex

@@ -95,18 +95,10 @@ defmodule Explorer.Chain.Import.Addresses do
           # MAX on two columns
           fetched_coin_balance_block_number:
             fragment(
-              """
+              "GREATEST(EXCLUDED.fetched_coin_balance_block_number, ?)",
-              CASE WHEN EXCLUDED.fetched_coin_balance_block_number IS NOT NULL AND
-                        (? IS NULL OR
-                         EXCLUDED.fetched_coin_balance_block_number >= ?) THEN
-                          EXCLUDED.fetched_coin_balance_block_number
-                   ELSE ?
-              END
-              """,
-              address.fetched_coin_balance_block_number,
-              address.fetched_coin_balance_block_number,
               address.fetched_coin_balance_block_number
-            )
+            ),
+          nonce: fragment("GREATEST(EXCLUDED.nonce, ?)", address.nonce)
         ]
       ]
     )

apps/explorer/priv/repo/migrations/20181106152300_add_nonce_to_addresses.exs

@@ -0,0 +1,26 @@
+defmodule Explorer.Repo.Migrations.AddNonceToAddresses do
+  @moduledoc """
+  Use `priv/repo/migrations/scripts/20181126182700_migrate_address_nonce.sql` to migrate data.
+
+  ```sh
+  mix ecto.migrate
+  psql -d $DATABASE -a -f priv/repo/migrations/scripts/20181126182700_migrate_address_nonce.sql
+  ```
+  """
+
+  use Ecto.Migration
+
+  def up do
+    # Add nonce
+    alter table(:addresses) do
+      add(:nonce, :integer)
+    end
+  end
+
+  def down do
+    # Remove nonce
+    alter table(:addresses) do
+      remove(:nonce)
+    end
+  end
+end

apps/explorer/priv/repo/migrations/20181126203826_add_index_to_addresses.exs

@@ -0,0 +1,13 @@
+defmodule Explorer.Repo.Migrations.AddIndexToAddresses do
+  use Ecto.Migration
+
+  def up do
+    execute(
+      "CREATE INDEX addresses_fetched_coin_balance_hash_index ON addresses (fetched_coin_balance DESC, hash ASC) WHERE fetched_coin_balance > 0"
+    )
+  end
+
+  def down do
+    execute("DROP INDEX addresses_fetched_coin_balance_hash_index")
+  end
+end

apps/explorer/priv/repo/migrations/scripts/20181126182700_migrate_address_nonce.sql

@@ -0,0 +1,55 @@
+DO $$
+ DECLARE
+    row_count integer := 1;
+    batch_size  integer := 50000; -- HOW MANY ITEMS WILL BE UPDATED AT TIME
+    iterator  integer := batch_size;
+    affected integer;
+ BEGIN
+   DROP TABLE IF EXISTS addresses_nonce_temp;
+
+   -- CREATES TEMP TABLE TO STORE THE ADDRESS NONCE TO BE UPDATED
+   CREATE TEMP TABLE addresses_nonce_temp(
+     from_address_hash bytea,
+     nonce integer,
+     row_number integer
+   );
+
+   INSERT INTO addresses_nonce_temp
+   SELECT DISTINCT ON (from_address_hash)
+     from_address_hash,
+     nonce,
+     ROW_NUMBER () OVER ()
+   FROM transactions
+   ORDER BY from_address_hash, nonce DESC;
+
+   row_count := (SELECT count(*) FROM addresses_nonce_temp);
+
+   RAISE NOTICE '% items to be updated', row_count;
+
+   -- ITERATES THROUGH THE ITEMS UNTIL THE TEMP TABLE IS EMPTY
+   WHILE row_count > 0 LOOP
+     -- UPDATES THE ADDRESS NONCE AND RETURNS THE ADDRESS_HASH
+     WITH updated_addresses AS (
+       UPDATE addresses SET nonce = addresses_nonce_temp.nonce
+       FROM addresses_nonce_temp
+       WHERE addresses_nonce_temp.from_address_hash = addresses.hash
+       AND addresses_nonce_temp.row_number <= iterator
+       RETURNING addresses_nonce_temp.from_address_hash
+     )
+     DELETE FROM addresses_nonce_temp
+     WHERE (from_address_hash) IN (select from_address_hash from updated_addresses);
+
+     GET DIAGNOSTICS affected = ROW_COUNT;
+     RAISE NOTICE '-> % addresses updated!', affected;
+
+     -- COMMITS THE BATCH UPDATES
+     CHECKPOINT;
+
+     -- UPDATES THE COUNTER SO IT DOESN'T TURN INTO A INFINITE LOOP
+     row_count := (SELECT count(*) FROM addresses_nonce_temp);
+     iterator := iterator + batch_size;
+
+     RAISE NOTICE '-> % counter', row_count;
+     RAISE NOTICE '-> % next batch', iterator;
+   END LOOP;
+ END $$;

apps/indexer/lib/indexer/address_extraction.ex

@@ -84,7 +84,8 @@ defmodule Indexer.AddressExtraction do
       ],
       [
         %{from: :block_number, to: :fetched_coin_balance_block_number},
-        %{from: :from_address_hash, to: :hash}
+        %{from: :from_address_hash, to: :hash},
+        %{from: :nonce, to: :nonce}
       ],
       [
         %{from: :block_number, to: :fetched_coin_balance_block_number},
@@ -136,6 +137,7 @@ defmodule Indexer.AddressExtraction do
           required(:hash) => String.t(),
           required(:fetched_coin_balance_block_number) => non_neg_integer(),
           optional(:fetched_coin_balance) => non_neg_integer(),
+          optional(:nonce) => non_neg_integer(),
           optional(:contract_code) => String.t()
         }
 
@@ -209,11 +211,13 @@ defmodule Indexer.AddressExtraction do
       ...>       %{
       ...>         block_number: 1,
       ...>         from_address_hash: "0x0000000000000000000000000000000000000001",
-      ...>         to_address_hash: "0x0000000000000000000000000000000000000002"
+      ...>         to_address_hash: "0x0000000000000000000000000000000000000002",
+      ...>         nonce: 3
       ...>       },
       ...>       %{
       ...>         block_number: 2,
-      ...>         from_address_hash: "0x0000000000000000000000000000000000000003"
+      ...>         from_address_hash: "0x0000000000000000000000000000000000000003",
+      ...>         nonce: 4
       ...>       }
       ...>     ]
       ...>   }
@@ -221,7 +225,8 @@ defmodule Indexer.AddressExtraction do
       [
         %{
           fetched_coin_balance_block_number: 1,
-          hash: "0x0000000000000000000000000000000000000001"
+          hash: "0x0000000000000000000000000000000000000001",
+          nonce: 3
         },
         %{
           fetched_coin_balance_block_number: 1,
@@ -229,7 +234,8 @@ defmodule Indexer.AddressExtraction do
         },
         %{
           fetched_coin_balance_block_number: 2,
-          hash: "0x0000000000000000000000000000000000000003"
+          hash: "0x0000000000000000000000000000000000000003",
+          nonce: 4
         }
       ]
 
@@ -279,11 +285,13 @@ defmodule Indexer.AddressExtraction do
       ...>     transactions: [
       ...>       %{
       ...>         block_number: 3,
-      ...>         to_address_hash: "0x0000000000000000000000000000000000000001"
+      ...>         to_address_hash: "0x0000000000000000000000000000000000000001",
+      ...>         nonce: 5
       ...>       },
       ...>       %{
       ...>         block_number: 2,
-      ...>         from_address_hash: "0x0000000000000000000000000000000000000001"
+      ...>         from_address_hash: "0x0000000000000000000000000000000000000001",
+      ...>         nonce: 4
       ...>       }
       ...>     ],
       ...>     logs: [
@@ -297,7 +305,8 @@ defmodule Indexer.AddressExtraction do
       [
         %{
           fetched_coin_balance_block_number: 7,
-          hash: "0x0000000000000000000000000000000000000001"
+          hash: "0x0000000000000000000000000000000000000001",
+          nonce: 4
         }
       ]
 
@@ -316,11 +325,13 @@ defmodule Indexer.AddressExtraction do
       ...>     transactions: [
       ...>       %{
       ...>         block_number: 2,
-      ...>         from_address_hash: "0x0000000000000000000000000000000000000001"
+      ...>         from_address_hash: "0x0000000000000000000000000000000000000001",
+      ...>         nonce: 4
       ...>       },
       ...>       %{
       ...>         block_number: 3,
-      ...>         to_address_hash: "0x0000000000000000000000000000000000000001"
+      ...>         to_address_hash: "0x0000000000000000000000000000000000000001",
+      ...>         nonce: 5
       ...>       }
       ...>     ]
       ...>   }
@@ -329,7 +340,8 @@ defmodule Indexer.AddressExtraction do
         %{
           contract_code: "0x",
           fetched_coin_balance_block_number: 3,
-          hash: "0x0000000000000000000000000000000000000001"
+          hash: "0x0000000000000000000000000000000000000001",
+          nonce: 4
         }
       ]
 
@@ -366,6 +378,7 @@ defmodule Indexer.AddressExtraction do
             %{
               required(:block_number) => non_neg_integer(),
               required(:from_address_hash) => String.t(),
+              required(:nonce) => non_neg_integer(),
               optional(:to_address_hash) => String.t(),
               optional(:created_contract_address_hash) => String.t()
             }
@@ -445,6 +458,7 @@ defmodule Indexer.AddressExtraction do
 
   # Ensure that when `:addresses` or `:address_coin_balances` are present, their :fetched_coin_balance will win
   defp merge_addresses(%{hash: hash} = first, %{hash: hash} = second) do
+    merged_addresses =
       case {first[:fetched_coin_balance], second[:fetched_coin_balance]} do
         {nil, nil} ->
           first
@@ -477,6 +491,9 @@ defmodule Indexer.AddressExtraction do
             Map.merge(first, second)
           end
       end
+
+    merged_addresses
+    |> Map.put(:nonce, max_nil_last(first[:nonce], second[:nonce]))
   end
 
   # `nil > 5 == true`, but we want numbers instead

apps/indexer/test/indexer/address_extraction_test.exs

@@ -76,7 +76,8 @@ defmodule Indexer.AddressExtractionTest do
                %{
                  fetched_coin_balance_block_number: 2,
                  contract_code: "0x1",
-                 hash: "0x0000000000000000000000000000000000000001"
+                 hash: "0x0000000000000000000000000000000000000001",
+                 nonce: nil
                }
              ]
     end
@@ -95,7 +96,8 @@ defmodule Indexer.AddressExtractionTest do
       transaction = %{
         block_number: 3,
         from_address_hash: gen_hash(),
-        to_address_hash: gen_hash()
+        to_address_hash: gen_hash(),
+        nonce: 7
       }
 
       log = %{address_hash: gen_hash(), block_number: 4}
@@ -136,7 +138,11 @@ defmodule Indexer.AddressExtractionTest do
                  contract_code: internal_transaction.created_contract_code,
                  fetched_coin_balance_block_number: internal_transaction.block_number
                },
-               %{hash: transaction.from_address_hash, fetched_coin_balance_block_number: transaction.block_number},
+               %{
+                 hash: transaction.from_address_hash,
+                 fetched_coin_balance_block_number: transaction.block_number,
+                 nonce: 7
+               },
                %{hash: transaction.to_address_hash, fetched_coin_balance_block_number: transaction.block_number},
                %{hash: log.address_hash, fetched_coin_balance_block_number: log.block_number},
                %{
@@ -176,7 +182,7 @@ defmodule Indexer.AddressExtractionTest do
 
       blockchain_data = %{
         blocks: [%{miner_hash: hash, number: 34}],
-        transactions: [%{block_number: 34, from_address_hash: hash}],
+        transactions: [%{block_number: 34, from_address_hash: hash, nonce: 12}],
         internal_transactions: [
           %{
             block_number: 34,
@@ -188,7 +194,7 @@ defmodule Indexer.AddressExtractionTest do
 
       assert AddressExtraction.extract_addresses(blockchain_data) ==
                [
-                 %{hash: hash, fetched_coin_balance_block_number: 34, contract_code: "code"}
+                 %{hash: hash, fetched_coin_balance_block_number: 34, contract_code: "code", nonce: 12}
                ]
     end