From 7d346d9975d84f36018047645f456bcea8650655 Mon Sep 17 00:00:00 2001
From: pasqu4le <pasqu4le@gmail.com>
Date: Fri, 18 Oct 2019 15:59:33 +0200
Subject: [PATCH] Add pending block operations

Problem: a lot of problems are created by tracking the importing progress on multiple tables.
Querying and updating involves as such heavy operations and this is also the cause for inconsistencies.

Solution: use a new table to track the progress of block's data indexing.
By using a specific table, whose rows are deleted when there are no more pending operations on the relative block, we make joins quick and keep them to a minimum.
This is also meant to simplify all inserts and updates and only perform deletes when strictly necessary.
---
 apps/explorer/lib/explorer/chain/block.ex     |  4 +-
 .../explorer/chain/pending_block_operation.ex | 46 +++++++++++++++++++
 ...120546_create_pending_block_operations.exs | 14 ++++++
 3 files changed, 63 insertions(+), 1 deletion(-)
 create mode 100644 apps/explorer/lib/explorer/chain/pending_block_operation.ex
 create mode 100644 apps/explorer/priv/repo/migrations/20191018120546_create_pending_block_operations.exs

diff --git a/apps/explorer/lib/explorer/chain/block.ex b/apps/explorer/lib/explorer/chain/block.ex
index 32905cf076..0035eb2688 100644
--- a/apps/explorer/lib/explorer/chain/block.ex
+++ b/apps/explorer/lib/explorer/chain/block.ex
@@ -7,7 +7,7 @@ defmodule Explorer.Chain.Block do
 
   use Explorer.Schema
 
-  alias Explorer.Chain.{Address, Gas, Hash, Transaction}
+  alias Explorer.Chain.{Address, Gas, Hash, PendingBlockOperation, Transaction}
   alias Explorer.Chain.Block.{Reward, SecondDegreeRelation}
 
   @optional_attrs ~w(internal_transactions_indexed_at size refetch_needed total_difficulty difficulty)a
@@ -97,6 +97,8 @@ defmodule Explorer.Chain.Block do
     has_many(:transaction_forks, Transaction.Fork, foreign_key: :uncle_hash)
 
     has_many(:rewards, Reward, foreign_key: :block_hash)
+
+    has_one(:pending_operations, PendingBlockOperation, foreign_key: :block_hash)
   end
 
   def changeset(%__MODULE__{} = block, attrs) do
diff --git a/apps/explorer/lib/explorer/chain/pending_block_operation.ex b/apps/explorer/lib/explorer/chain/pending_block_operation.ex
new file mode 100644
index 0000000000..fa6ad4c2a5
--- /dev/null
+++ b/apps/explorer/lib/explorer/chain/pending_block_operation.ex
@@ -0,0 +1,46 @@
+defmodule Explorer.Chain.PendingBlockOperation do
+  @moduledoc """
+  Tracks a block that has pending operations.
+  """
+
+  use Explorer.Schema
+
+  alias Explorer.Chain.{Block, Hash}
+
+  @required_attrs ~w(block_hash)a
+
+  @typedoc """
+   * `block_hash` - the hash of the block that has pending operations.
+  """
+  @type t :: %__MODULE__{
+          block_hash: Hash.Full.t()
+        }
+
+  @primary_key false
+  schema "pending_block_operations" do
+    timestamps()
+
+    belongs_to(:block, Block, foreign_key: :block_hash, primary_key: true, references: :hash, type: Hash.Full)
+  end
+
+  def changeset(%__MODULE__{} = pending_ops, attrs) do
+    pending_ops
+    |> cast(attrs, @required_attrs)
+    |> validate_required(@required_attrs)
+    |> foreign_key_constraint(:block_hash)
+    |> unique_constraint(:block_hash, name: :pending_block_operations_pkey)
+  end
+
+  @doc """
+  Returns all pending block operations with the `block_hash` in the given list,
+  using "FOR UPDATE" to grab ShareLocks in order (see docs: sharelocks.md)
+  """
+  def fetch_and_lock_by_hashes(hashes) when is_list(hashes) do
+    from(
+      pending_ops in __MODULE__,
+      where: pending_ops.block_hash in ^hashes,
+      order_by: [asc: pending_ops.block_hash],
+      lock: "FOR UPDATE"
+    )
+  end
+end
diff --git a/apps/explorer/priv/repo/migrations/20191018120546_create_pending_block_operations.exs b/apps/explorer/priv/repo/migrations/20191018120546_create_pending_block_operations.exs
new file mode 100644
index 0000000000..3e917122a6
--- /dev/null
+++ b/apps/explorer/priv/repo/migrations/20191018120546_create_pending_block_operations.exs
@@ -0,0 +1,14 @@
+defmodule Explorer.Repo.Migrations.CreatePendingBlockOperations do
+  use Ecto.Migration
+
+  def change do
+    create table(:pending_block_operations, primary_key: false) do
+      add(:block_hash, references(:blocks, column: :hash, type: :bytea, on_delete: :delete_all),
+        null: false,
+        primary_key: true
+      )
+
+      timestamps(null: false, type: :utc_datetime_usec)
+    end
+  end
+end