Denormalize blocks.number into transasctions.block_number

Allows to use a sorted index for
`Explorer.Chain.recent_collated_transactions`, which speeds up the query
600x with 1000 blocks with 100 transaction each and the speed up appears
to only grow as more blocks and transactions per block is used.

apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/transaction.ex

@@ -48,6 +48,7 @@ defmodule EthereumJSONRPC.Transaction do
 
   @type params :: %{
           block_hash: EthereumJSONRPC.hash(),
+          block_number: non_neg_integer(),
           from_address_hash: EthereumJSONRPC.address(),
           gas: non_neg_integer(),
           gas_price: non_neg_integer(),
@@ -67,6 +68,7 @@ defmodule EthereumJSONRPC.Transaction do
   @spec elixir_to_params(elixir) :: params
   def elixir_to_params(%{
         "blockHash" => block_hash,
+        "blockNumber" => block_number,
         "from" => from_address_hash,
         "gas" => gas,
         "gasPrice" => gas_price,
@@ -84,6 +86,7 @@ defmodule EthereumJSONRPC.Transaction do
       }) do
     %{
       block_hash: block_hash,
+      block_number: block_number,
       from_address_hash: from_address_hash,
       gas: gas,
       gas_price: gas_price,
@@ -107,6 +110,7 @@ defmodule EthereumJSONRPC.Transaction do
       iex> EthereumJSONRPC.Transaction.params_to_hash(
       ...>   %{
       ...>     block_hash: "0xe52d77084cab13a4e724162bcd8c6028e5ecfaa04d091ee476e96b9958ed6b47",
+      ...>     block_number: 34,
       ...>     gas: 4700000,
       ...>     gas_price: 100000000000,
       ...>     hash: "0x3a3eb134e6792ce9403ea4188e5e79693de9e4c94e499db132be086400da79e6",

apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/transactions.ex

@@ -42,6 +42,7 @@ defmodule EthereumJSONRPC.Transactions do
       [
         %{
           block_hash: "0xe52d77084cab13a4e724162bcd8c6028e5ecfaa04d091ee476e96b9958ed6b47",
+          block_number: 34,
           from_address_hash: "0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca",
           gas: 4700000,
           gas_price: 100000000000,
@@ -71,6 +72,7 @@ defmodule EthereumJSONRPC.Transactions do
       ...>   [
       ...>     %{
       ...>       block_hash: "0xe52d77084cab13a4e724162bcd8c6028e5ecfaa04d091ee476e96b9958ed6b47",
+      ...>       block_number: 34,
       ...>       from_address_hash: "0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca",
       ...>       gas: 4700000,
       ...>       gas_price: 100000000000,

apps/explorer/README.md

@@ -39,4 +39,5 @@ To get POA Explorer up and running locally:
 #### `Explorer.Chain.recent_collated_transactions/0`
 
 * Reset the test database: `MIX_ENV=test mix do ecto.drop, ecto.create, ecto.migrate`
+* Change `tag` in `benchmarks/explorer/chain/recent_collated_transactions.exs` to a new value, so that it will compare against the old values saved in `benchmarks/explorer/chain/recent_collated_transactions.benchee`
 * Run the benchmark: `MIX_ENV=test mix run benchmarks/explorer/chain/recent_collated_transactions.exs`

apps/explorer/benchmarks/explorer/chain/recent_collated_transactions.exs

@@ -50,7 +50,7 @@ Benchee.run(
   load: path,
   save: [
     path: path,
-    tag: "transactions-index-index"
+    tag: "transactions-block-number"
   ],
   time: 10
 )

apps/explorer/lib/explorer/chain.ex

@@ -634,6 +634,7 @@ defmodule Explorer.Chain do
       ...>     transactions: [
       ...>       %{
       ...>         block_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd",
+      ...>         block_number: 37,
       ...>         cumulative_gas_used: 50450,
       ...>         from_address_hash: "0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca",
       ...>         gas: 4700000,
@@ -798,6 +799,7 @@ defmodule Explorer.Chain do
       ...>     transactions: [
       ...>       %{
       ...>         block_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd",
+      ...>         block_number: 37,
       ...>         cumulative_gas_used: 50450,
       ...>         from_address_hash: "0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca",
       ...>         gas: 4700000,
@@ -1179,8 +1181,8 @@ defmodule Explorer.Chain do
     query =
       from(
         transaction in Transaction,
-        inner_join: block in assoc(transaction, :block),
-        order_by: [desc: block.number, desc: transaction.index],
+        where: not is_nil(transaction.block_number) and not is_nil(transaction.index),
+        order_by: [desc: transaction.block_number, desc: transaction.index],
         limit: 10
       )
 

apps/explorer/lib/explorer/chain/transaction.ex

@@ -7,7 +7,8 @@ defmodule Explorer.Chain.Transaction do
   alias Explorer.Chain.{Address, Block, Data, Gas, Hash, InternalTransaction, Log, Wei}
   alias Explorer.Chain.Transaction.Status
 
-  @optional_attrs ~w(block_hash cumulative_gas_used from_address_hash gas_used index status to_address_hash)a
+  @optional_attrs ~w(block_hash block_number cumulative_gas_used from_address_hash gas_used index status
+                     to_address_hash)a
   @required_attrs ~w(gas gas_price hash input nonce public_key r s standard_v v value)a
 
   @typedoc """
@@ -74,6 +75,7 @@ defmodule Explorer.Chain.Transaction do
   @typedoc """
    * `block` - the block in which this transaction was mined/validated.  `nil` when transaction is pending.
    * `block_hash` - `block` foreign key. `nil` when transaction is pending.
+   * `block_number` - Denormalized `block` `number`. `nil` when transaction is pending.
    * `cumulative_gas_used` - the cumulative gas used in `transaction`'s `t:Explorer.Chain.Block.t/0` before
      `transaction`'s `index`.  `nil` when transaction is pending.
    * `from_address` - the source of `value`
@@ -103,6 +105,7 @@ defmodule Explorer.Chain.Transaction do
   @type t :: %__MODULE__{
           block: %Ecto.Association.NotLoaded{} | Block.t() | nil,
           block_hash: Hash.t() | nil,
+          block_number: Block.block_number() | nil,
           cumulative_gas_used: Gas.t() | nil,
           from_address: %Ecto.Association.NotLoaded{} | Address.t(),
           from_address_hash: Hash.Truncated.t(),
@@ -128,6 +131,7 @@ defmodule Explorer.Chain.Transaction do
 
   @primary_key {:hash, Hash.Full, autogenerate: false}
   schema "transactions" do
+    field(:block_number, :integer)
     field(:cumulative_gas_used, :decimal)
     field(:gas, :decimal)
     field(:gas_price, Wei)
@@ -189,12 +193,13 @@ defmodule Explorer.Chain.Transaction do
       iex> changeset.valid?
       true
 
-  A pending transaction (which is indicated by not having a `block_hash`) can't have `cumulative_gas_used`, `gas_used`,
-  or `index`.
+  A pending transaction (which is indicated by not having a `block_hash`) can't have `block_number`,
+  `cumulative_gas_used`, `gas_used`, or `index`.
 
       iex> changeset = Explorer.Chain.Transaction.changeset(
       ...>   %Transaction{},
       ...>   %{
+      ...>     block_number: 34,
       ...>     cumulative_gas_used: 0,
       ...>     gas: 4700000,
       ...>     gas_price: 100000000000,
@@ -214,6 +219,8 @@ defmodule Explorer.Chain.Transaction do
       ...> )
       iex> changeset.valid?
       false
+      iex> Keyword.get_values(changeset.errors, :block_number)
+      [{"can't be set when the transaction is pending", []}]
       iex> Keyword.get_values(changeset.errors, :cumulative_gas_used)
       [{"can't be set when the transaction is pending", []}]
       iex> Keyword.get_values(changeset.errors, :gas_used)
@@ -229,6 +236,7 @@ defmodule Explorer.Chain.Transaction do
       ...>   %Transaction{},
       ...>   %{
       ...>     block_hash: "0xe52d77084cab13a4e724162bcd8c6028e5ecfaa04d091ee476e96b9958ed6b47",
+      ...>     block_number: 34,
       ...>     cumulative_gas_used: 0,
       ...>     gas: 4700000,
       ...>     gas_price: 100000000000,
@@ -271,6 +279,8 @@ defmodule Explorer.Chain.Transaction do
       ...> )
       iex> changeset.valid?
       false
+      iex> Keyword.get_values(changeset.errors, :block_number)
+      [{"can't be blank when the transaction is collated into a block", []}]
       iex> Keyword.get_values(changeset.errors, :cumulative_gas_used)
       [{"can't be blank when the transaction is collated into a block", []}]
       iex> Keyword.get_values(changeset.errors, :gas_used)
@@ -293,7 +303,7 @@ defmodule Explorer.Chain.Transaction do
     |> unique_constraint(:hash)
   end
 
-  @collated_fields ~w(cumulative_gas_used gas_used index status)a
+  @collated_fields ~w(block_number cumulative_gas_used gas_used index status)a
 
   @collated_message "can't be blank when the transaction is collated into a block"
   @collated_field_to_check Enum.into(@collated_fields, %{}, fn collated_field ->

apps/explorer/priv/repo/migrations/20180117221923_create_transactions.exs

@@ -35,11 +35,23 @@ defmodule Explorer.Repo.Migrations.CreateTransactions do
       # `null` when a pending transaction
       add(:block_hash, references(:blocks, column: :hash, on_delete: :delete_all, type: :bytea), null: true)
 
+      # `null` when a pending transaction
+      # denormalized from `blocks.number` to improve `Explorer.Chain.recent_collated_transactions/0` performance
+      add(:block_number, :integer, null: true)
+
       add(:from_address_hash, references(:addresses, column: :hash, on_delete: :delete_all, type: :bytea), null: false)
       # `null` when it is a contract creation transaction
       add(:to_address_hash, references(:addresses, column: :hash, on_delete: :delete_all, type: :bytea), null: true)
     end
 
+    create(
+      constraint(
+        :transactions,
+        :collated_block_number,
+        check: "block_hash IS NULL OR block_number IS NOT NULL"
+      )
+    )
+
     create(
       constraint(
         :transactions,
@@ -72,6 +84,14 @@ defmodule Explorer.Repo.Migrations.CreateTransactions do
       )
     )
 
+    create(
+      constraint(
+        :transactions,
+        :pending_block_number,
+        check: "block_hash IS NOT NULL OR block_number IS NULL"
+      )
+    )
+
     create(
       constraint(
         :transactions,
@@ -114,7 +134,14 @@ defmodule Explorer.Repo.Migrations.CreateTransactions do
     create(index(:transactions, :updated_at))
 
     create(index(:transactions, :status))
-    create(index(:transactions, ["index DESC NULLS FIRST"], name: "transactions_index_index"))
+
+    create(
+      index(
+        :transactions,
+        ["block_number DESC NULLS FIRST", "index DESC NULLS FIRST"],
+        name: "transactions_recent_collated_index"
+      )
+    )
 
     create(unique_index(:transactions, [:block_hash, :index]))
   end

apps/explorer/test/support/factory.ex

@@ -85,7 +85,11 @@ defmodule Explorer.Factory do
     with_block(transaction, block, collated_params)
   end
 
-  def with_block(%Transaction{index: nil} = transaction, %Block{hash: block_hash}, collated_params)
+  def with_block(
+        %Transaction{index: nil} = transaction,
+        %Block{hash: block_hash, number: block_number},
+        collated_params
+      )
       when is_list(collated_params) do
     next_transaction_index = block_hash_to_next_transaction_index(block_hash)
 
@@ -96,6 +100,7 @@ defmodule Explorer.Factory do
     transaction
     |> Transaction.changeset(%{
       block_hash: block_hash,
+      block_number: block_number,
       cumulative_gas_used: cumulative_gas_used,
       gas_used: gas_used,
       index: next_transaction_index,