Merge pull request #753 from poanetwork/optimize-address-page-loading

Optimize address page loading

apps/block_scout_web/lib/block_scout_web/controllers/address_controller.ex

@@ -9,6 +9,6 @@ defmodule BlockScoutWeb.AddressController do
   end
 
   def transaction_count(%Address{} = address) do
-    Chain.address_to_transaction_count(address)
+    Chain.address_to_transaction_count_estimate(address)
   end
 end

apps/block_scout_web/lib/block_scout_web/templates/address/overview.html.eex

@@ -22,18 +22,18 @@
             <%= if address_name = primary_name(@address) do %>
               <strong class="mr-4 mb-2 text-primary"><%= address_name %></strong>
             <% end %>
-            <span class="mb-2">
+            <span class="mr-4 mb-2">
               <span data-selector="transaction-count"><%= Cldr.Number.to_string!(@transaction_count, format: "#,###") %></span>
 
               <%= gettext "Transactions" %>
             </span>
             <%= if @address.token do %>
-              <span class="mb-2">
+              <span class="mr-4 mb-2">
                 <%= link(token_title(@address.token), to: token_path(@conn, :show, @address.hash), "data-test": "token_hash_link" ) %>
               </span>
             <% end %>
             <%= if contract?(@address) do %>
-              <span class="ml-lg-4" data-test="address_contract_creator">
+              <span data-test="address_contract_creator">
                 <%= gettext "Contract created by" %>
                 <%= link(
                   trimmed_hash(@address.contracts_creation_internal_transaction.from_address_hash),

apps/block_scout_web/test/block_scout_web/features/viewing_addresses_test.exs

@@ -241,14 +241,6 @@ defmodule BlockScoutWeb.ViewingAddressesTest do
     end
   end
 
-  test "viewing transaction count", %{addresses: addresses, session: session} do
-    insert_list(1000, :transaction, to_address: addresses.lincoln)
-
-    session
-    |> AddressPage.visit_page(addresses.lincoln)
-    |> assert_text(AddressPage.transaction_count(), "1,002")
-  end
-
   test "contract creation is shown for to_address on list page", %{
     addresses: addresses,
     block: block,

apps/explorer/lib/explorer/chain.ex

@@ -132,33 +132,35 @@ defmodule Explorer.Chain do
   end
 
   @doc """
-  Counts the number of `t:Explorer.Chain.Transaction.t/0` to or from the `address`.
+  Gets an estimated count of `t:Explorer.Chain.Transaction.t/0` to or from the `address` based on the estimated rows
+  resulting in an EXPLAIN of the query plan for the count query.
   """
-  @spec address_to_transaction_count(Address.t()) :: non_neg_integer()
-  def address_to_transaction_count(%Address{hash: hash}) do
-    {:ok, %{rows: [[result]]}} =
-      SQL.query(
-        Repo,
+  @spec address_to_transaction_count_estimate(Address.t()) :: non_neg_integer()
+  def address_to_transaction_count_estimate(%Address{hash: address_hash}) do
+    {:ok, %Postgrex.Result{rows: result}} =
+      Repo.query(
         """
-          SELECT COUNT(hash) from
-          (
-            SELECT t0."hash" address
-            FROM "transactions" AS t0
-            LEFT OUTER JOIN "internal_transactions" AS i1 ON (i1."transaction_hash" = t0."hash") AND (i1."type" = 'create')
-            WHERE (i1."created_contract_address_hash" = $1 AND t0."to_address_hash" IS NULL)
-
-            UNION
-
-            SELECT t0."hash" address
-            FROM "transactions" AS t0
-            WHERE (t0."to_address_hash" = $1)
-            OR (t0."from_address_hash" = $1)
-          ) AS hash
+        EXPLAIN SELECT COUNT(DISTINCT t.hash) FROM
+        (
+          SELECT t0.hash FROM transactions AS t0 WHERE t0.from_address_hash = $1
+          UNION
+          SELECT t0.hash FROM transactions AS t0 WHERE t0.to_address_hash = $1
+          UNION
+          SELECT t0.hash FROM transactions AS t0 WHERE t0.created_contract_address_hash = $1
+          UNION
+          SELECT tt.transaction_hash AS hash FROM token_transfers AS tt
+          WHERE tt.from_address_hash = $1
+          UNION
+          SELECT tt.transaction_hash AS hash FROM token_transfers AS tt
+          WHERE tt.to_address_hash = $1
+        ) as t
         """,
-        [hash.bytes]
+        [address_hash.bytes]
       )
 
-    result
+    {[unique_explain], _} = List.pop_at(result, 1)
+    [[_ | [rows]]] = Regex.scan(~r/rows=(\d+)/, unique_explain)
+    String.to_integer(rows)
   end
 
   @doc """
@@ -182,14 +184,40 @@ defmodule Explorer.Chain do
       when is_list(options) do
     direction = Keyword.get(options, :direction)
     necessity_by_association = Keyword.get(options, :necessity_by_association, %{})
+    paging_options = Keyword.get(options, :paging_options, @default_paging_options)
 
-    options
-    |> Keyword.get(:paging_options, @default_paging_options)
-    |> fetch_transactions()
-    |> Transaction.where_address_fields_match(address_hash, direction)
-    |> join_associations(necessity_by_association)
-    |> Transaction.preload_token_transfers(address_hash)
-    |> Repo.all()
+    transaction_matches =
+      direction
+      |> case do
+        :from -> [:from_address_hash]
+        :to -> [:to_address_hash, :created_contract_address_hash]
+        _ -> [:from_address_hash, :to_address_hash, :created_contract_address_hash]
+      end
+      |> Enum.map(fn address_field ->
+        paging_options
+        |> fetch_transactions()
+        |> Transaction.where_address_fields_match(address_hash, address_field)
+        |> join_associations(necessity_by_association)
+        |> Transaction.preload_token_transfers(address_hash)
+        |> Repo.all()
+        |> MapSet.new()
+      end)
+
+    token_transfer_matches =
+      paging_options
+      |> fetch_transactions()
+      |> TokenTransfer.where_address_fields_match(address_hash, direction)
+      |> join_associations(necessity_by_association)
+      |> Transaction.preload_token_transfers(address_hash)
+      |> Repo.all()
+      |> MapSet.new()
+
+    transaction_matches
+    |> Enum.reduce(token_transfer_matches, &MapSet.union/2)
+    |> MapSet.to_list()
+    |> Enum.sort_by(& &1.index, &>=/2)
+    |> Enum.sort_by(& &1.block_number, &>=/2)
+    |> Enum.slice(0..paging_options.page_size)
   end
 
   @doc """

apps/explorer/lib/explorer/chain/token_transfer.ex

@@ -148,4 +148,22 @@ defmodule Explorer.Chain.TokenTransfer do
       token_transfer.inserted_at < ^inserted_at
     )
   end
+
+  def where_address_fields_match(query, address_hash, :from) do
+    query
+    |> join(:left, [transaction], tt in assoc(transaction, :token_transfers))
+    |> where([_transaction, tt], tt.from_address_hash == ^address_hash)
+  end
+
+  def where_address_fields_match(query, address_hash, :to) do
+    query
+    |> join(:left, [transaction], tt in assoc(transaction, :token_transfers))
+    |> where([_transaction, tt], tt.to_address_hash == ^address_hash)
+  end
+
+  def where_address_fields_match(query, address_hash, _) do
+    query
+    |> join(:left, [transaction], tt in assoc(transaction, :token_transfers))
+    |> where([_transaction, tt], tt.to_address_hash == ^address_hash or tt.from_address_hash == ^address_hash)
+  end
 end

apps/explorer/lib/explorer/chain/transaction.ex

@@ -3,7 +3,7 @@ defmodule Explorer.Chain.Transaction do
 
   use Explorer.Schema
 
-  import Ecto.Query, only: [from: 2, join: 5, preload: 3]
+  import Ecto.Query, only: [from: 2, preload: 3, where: 3]
 
   alias Ecto.Changeset
 
@@ -405,110 +405,12 @@ defmodule Explorer.Chain.Transaction do
   Adds to the given transaction's query a `where` with one of the conditions that the matched
   function returns.
 
-  `where_address_fields_match(query, address, :to)`
-  - returns a query considering that the given address_hash is equal to to_address_hash from
-    transactions' table or is equal to to_address_hash from token transfers' table.
-
-  `where_address_fields_match(query, address, :from)`
-  - returns a query considering that the given address_hash is equal to from_address_hash from
-    transactions' table or is equal to from_address_hash from token transfers' table.
-
-  `where_address_fields_match(query, address, nil)`
-  - returns a query considering that the given address_hash can be: to_address_hash,
-    from_address_hash, created_contract_address_hash,
-    to_address_hash or from_address_hash from token_transfers' table.
-
-  ### Token transfers' preload
-
-  Token transfers will be preloaded according to the given address_hash considering if it's equal
-  to token_contract_address_hash, to_address_hash or from_address_hash from Token Transfers's table.
+  `where_address_fields_match(query, address, address_field)`
+  - returns a query constraining the given address_hash to be equal to the given
+    address field from transactions' table.
   """
-  def where_address_fields_match(query, address_hash, :to) do
-    join(
-      query,
-      :inner,
-      [transaction],
-      matches in fragment(
-        """
-        WITH hashes AS (
-          (
-            SELECT t0.hash AS hash
-            FROM transactions AS t0
-            WHERE t0.to_address_hash = ? OR t0.created_contract_address_hash = ?
-          )
-          UNION ALL
-          (
-            SELECT tt.transaction_hash AS hash
-            FROM token_transfers AS tt
-            WHERE tt.to_address_hash = ?
-          )
-        ) SELECT * from hashes
-        """,
-        ^address_hash.bytes,
-        ^address_hash.bytes,
-        ^address_hash.bytes
-      ),
-      transaction.hash == matches.hash
-    )
-  end
-
-  def where_address_fields_match(query, address_hash, :from) do
-    join(
-      query,
-      :inner,
-      [transaction],
-      matches in fragment(
-        """
-        WITH hashes AS (
-          (
-            SELECT t0.hash AS hash
-            FROM transactions AS t0
-            WHERE t0.from_address_hash = ?
-          )
-          UNION ALL
-          (
-            SELECT tt.transaction_hash AS hash
-            FROM token_transfers AS tt
-            WHERE tt.from_address_hash = ?
-          )
-        ) SELECT * from hashes
-        """,
-        ^address_hash.bytes,
-        ^address_hash.bytes
-      ),
-      transaction.hash == matches.hash
-    )
-  end
-
-  def where_address_fields_match(query, address_hash, nil) do
-    join(
-      query,
-      :inner,
-      [transaction],
-      matches in fragment(
-        """
-        WITH hashes AS (
-          (
-            SELECT t0.hash AS hash
-            FROM transactions AS t0
-            WHERE t0.to_address_hash = ? OR t0.from_address_hash = ? OR t0.created_contract_address_hash = ?
-          )
-          UNION ALL
-          (
-            SELECT tt.transaction_hash AS hash
-            FROM token_transfers AS tt
-            WHERE tt.to_address_hash = ? OR tt.from_address_hash = ?
-          )
-        ) SELECT * from hashes
-        """,
-        ^address_hash.bytes,
-        ^address_hash.bytes,
-        ^address_hash.bytes,
-        ^address_hash.bytes,
-        ^address_hash.bytes
-      ),
-      transaction.hash == matches.hash
-    )
+  def where_address_fields_match(query, address_hash, address_field) do
+    where(query, [t], field(t, ^address_field) == ^address_hash)
   end
 
   @collated_fields ~w(block_number cumulative_gas_used gas_used index)a

apps/explorer/priv/repo/migrations/20180117221922_create_blocks.exs

@@ -4,8 +4,8 @@ defmodule Explorer.Repo.Migrations.CreateBlocks do
   def change do
     create table(:blocks, primary_key: false) do
       add(:difficulty, :numeric, precision: 50)
-      add(:gas_limit, :integer, null: false)
-      add(:gas_used, :integer, null: false)
+      add(:gas_limit, :numeric, precision: 100, null: false)
+      add(:gas_used, :numeric, precision: 100, null: false)
       add(:hash, :bytea, null: false, primary_key: true)
       add(:miner_hash, references(:addresses, column: :hash, type: :bytea), null: false)
       add(:nonce, :bytea, null: false)

apps/explorer/priv/repo/migrations/20180117221923_create_transactions.exs

@@ -182,9 +182,6 @@ defmodule Explorer.Repo.Migrations.CreateTransactions do
     )
 
     create(index(:transactions, :block_hash))
-    create(index(:transactions, :from_address_hash))
-    create(index(:transactions, :to_address_hash))
-    create(index(:transactions, :created_contract_address_hash))
 
     create(index(:transactions, :inserted_at))
     create(index(:transactions, :updated_at))
@@ -199,6 +196,30 @@ defmodule Explorer.Repo.Migrations.CreateTransactions do
       )
     )
 
+    create(
+      index(
+        :transactions,
+        [:from_address_hash, "block_number DESC NULLS FIRST", "index DESC NULLS FIRST", :hash],
+        name: "transactions_from_address_hash_recent_collated_index"
+      )
+    )
+
+    create(
+      index(
+        :transactions,
+        [:to_address_hash, "block_number DESC NULLS FIRST", "index DESC NULLS FIRST", :hash],
+        name: "transactions_to_address_hash_recent_collated_index"
+      )
+    )
+
+    create(
+      index(
+        :transactions,
+        [:created_contract_address_hash, "block_number DESC NULLS FIRST", "index DESC NULLS FIRST", :hash],
+        name: "transactions_created_contract_address_hash_recent_collated_index"
+      )
+    )
+
     create(unique_index(:transactions, [:block_hash, :index]))
   end
 end

apps/explorer/priv/repo/migrations/20180606135150_create_token_transfers.exs

@@ -23,9 +23,9 @@ defmodule Explorer.Repo.Migrations.CreateTokenTransfers do
     end
 
     create(index(:token_transfers, :transaction_hash))
-    create(index(:token_transfers, :to_address_hash))
-    create(index(:token_transfers, :from_address_hash))
-    create(index(:token_transfers, :token_contract_address_hash))
+    create(index(:token_transfers, [:to_address_hash, :transaction_hash]))
+    create(index(:token_transfers, [:from_address_hash, :transaction_hash]))
+    create(index(:token_transfers, [:token_contract_address_hash, :transaction_hash]))
 
     create(unique_index(:token_transfers, [:transaction_hash, :log_index]))
   end

apps/explorer/priv/repo/migrations/20180831231919_modify_block_gas.exs

@@ -1,10 +0,0 @@
-defmodule Explorer.Repo.Migrations.ModifyBlockGas do
-  use Ecto.Migration
-
-  def change do
-    alter table(:blocks) do
-      modify(:gas_used, :numeric, precision: 100)
-      modify(:gas_limit, :numeric, precision: 100)
-    end
-  end
-end

apps/explorer/test/explorer/chain_test.exs

@@ -29,54 +29,6 @@ defmodule Explorer.ChainTest do
     end
   end
 
-  describe "address_to_transaction_count/1" do
-    test "without transactions" do
-      address = insert(:address)
-
-      assert Chain.address_to_transaction_count(address) == 0
-    end
-
-    test "with transactions" do
-      %Transaction{from_address: address} = insert(:transaction) |> Repo.preload(:from_address)
-      insert(:transaction, to_address: address)
-
-      assert Chain.address_to_transaction_count(address) == 2
-    end
-
-    test "with contract creation transactions the contract address is counted" do
-      address = insert(:address)
-
-      insert(
-        :internal_transaction_create,
-        created_contract_address: address,
-        index: 0,
-        transaction: insert(:transaction, to_address: nil)
-      )
-
-      assert Chain.address_to_transaction_count(address) == 1
-    end
-
-    test "doesn't double count addresses when to_address = from_address" do
-      %Transaction{from_address: address} = insert(:transaction) |> Repo.preload(:from_address)
-      insert(:transaction, to_address: address, from_address: address)
-
-      assert Chain.address_to_transaction_count(address) == 2
-    end
-
-    test "does not count non-contract-creation parent transactions" do
-      transaction_with_to_address =
-        %Transaction{} =
-        :transaction
-        |> insert()
-        |> with_block()
-
-      %InternalTransaction{created_contract_address: address} =
-        insert(:internal_transaction_create, transaction: transaction_with_to_address, index: 0)
-
-      assert Chain.address_to_transaction_count(address) == 0
-    end
-  end
-
   describe "address_to_transactions/2" do
     test "without transactions" do
       address = insert(:address)
@@ -275,6 +227,54 @@ defmodule Explorer.ChainTest do
                |> Enum.map(& &1.hash)
                |> Enum.reverse()
     end
+
+    test "returns results in reverse chronological order by block number and transaction index" do
+      address = insert(:address)
+
+      %Transaction{hash: first_pending} = insert(:transaction, to_address: address)
+      %Transaction{hash: second_pending} = insert(:transaction, to_address: address)
+
+      a_block = insert(:block, number: 6000)
+
+      %Transaction{hash: first} =
+        :transaction
+        |> insert(to_address: address)
+        |> with_block(a_block)
+
+      %Transaction{hash: second} =
+        :transaction
+        |> insert(to_address: address)
+        |> with_block(a_block)
+
+      %Transaction{hash: third} =
+        :transaction
+        |> insert(to_address: address)
+        |> with_block(a_block)
+
+      %Transaction{hash: fourth} =
+        :transaction
+        |> insert(to_address: address)
+        |> with_block(a_block)
+
+      b_block = insert(:block, number: 2000)
+
+      %Transaction{hash: fifth} =
+        :transaction
+        |> insert(to_address: address)
+        |> with_block(b_block)
+
+      %Transaction{hash: sixth} =
+        :transaction
+        |> insert(to_address: address)
+        |> with_block(b_block)
+
+      result =
+        address
+        |> Chain.address_to_transactions()
+        |> Enum.map(& &1.hash)
+
+      assert [first_pending, second_pending, fourth, third, second, first, sixth, fifth] == result
+    end
   end
 
   describe "average_block_time/0" do