Drop unused token_id column from token_transfers table and indexes based on this column (#9005)

* Drop unused token_id column from token_transfers table and indexes on this column

* Remove token ids migration

* Refactor DB migration

* Update CHANGE LOG entry
v6.0.0-dev
Victor Baranov 10 months ago committed by Viktor Baranov
parent 83978978f9
commit a4799d35aa
  1. 1
      CHANGELOG.md
  2. 2
      apps/block_scout_web/lib/block_scout_web/models/transaction_state_helper.ex
  3. 1
      apps/block_scout_web/lib/block_scout_web/schema/types.ex
  4. 2
      apps/block_scout_web/test/block_scout_web/controllers/tokens/inventory_controller_test.exs
  5. 3
      apps/block_scout_web/test/block_scout_web/schema/query/token_transfers_test.exs
  6. 2
      apps/block_scout_web/test/block_scout_web/views/tokens/helper_test.exs
  7. 2
      apps/explorer/config/config.exs
  8. 2
      apps/explorer/config/runtime/test.exs
  9. 3
      apps/explorer/lib/explorer/application.ex
  10. 5
      apps/explorer/lib/explorer/chain/token_transfer.ex
  11. 2
      apps/explorer/lib/explorer/chain/transaction/state_change.ex
  12. 65
      apps/explorer/lib/explorer/token_transfer_token_id_migration/lowest_block_number_updater.ex
  13. 70
      apps/explorer/lib/explorer/token_transfer_token_id_migration/supervisor.ex
  14. 84
      apps/explorer/lib/explorer/token_transfer_token_id_migration/worker.ex
  15. 67
      apps/explorer/lib/explorer/utility/token_transfer_token_id_migrator_progress.ex
  16. 12
      apps/explorer/priv/repo/migrations/20231215094615_drop_token_transfers_token_id_column.exs
  17. 37
      apps/explorer/test/explorer/token_transfer_token_id_migration/lowest_block_number_updater_test.exs
  18. 31
      apps/explorer/test/explorer/token_transfer_token_id_migration/worker_test.exs

@ -11,6 +11,7 @@
- [#9009](https://github.com/blockscout/blockscout/pull/9009) - Index for block refetch_needed
- [#9007](https://github.com/blockscout/blockscout/pull/9007) - Drop logs type index
- [#9006](https://github.com/blockscout/blockscout/pull/9006) - Drop unused indexes on address_current_token_balances table
- [#9005](https://github.com/blockscout/blockscout/pull/9005) - Drop unused token_id column from token_transfers table and indexes based on this column
- [#9000](https://github.com/blockscout/blockscout/pull/9000) - Change log topic type in the DB to bytea
- [#8996](https://github.com/blockscout/blockscout/pull/8996) - Refine token transfers token ids index
- [#5322](https://github.com/blockscout/blockscout/pull/5322) - DB denormalization: block consensus and timestamp in transaction table

@ -112,7 +112,7 @@ defmodule BlockScoutWeb.Models.TransactionStateHelper do
token_ids =
if token.type == "ERC-1155" do
token_transfer.token_ids || [token_transfer.token_id]
token_transfer.token_ids
else
[nil]
end

@ -130,7 +130,6 @@ defmodule BlockScoutWeb.Schema.Types do
field(:amounts, list_of(:decimal))
field(:block_number, :integer)
field(:log_index, :integer)
field(:token_id, :decimal)
field(:token_ids, list_of(:decimal))
field(:from_address_hash, :address_hash)
field(:to_address_hash, :address_hash)

@ -126,7 +126,7 @@ defmodule BlockScoutWeb.Tokens.InventoryControllerTest do
transaction: transaction,
token_contract_address: token.contract_address,
token: token,
token_id: 1000
token_ids: [1000]
)
conn = get(conn, token_inventory_path(conn, :index, token.contract_address_hash), %{type: "JSON"})

@ -16,7 +16,6 @@ defmodule BlockScoutWeb.Schema.Query.TokenTransfersTest do
amounts
block_number
log_index
token_id
token_ids
from_address_hash
to_address_hash
@ -45,7 +44,6 @@ defmodule BlockScoutWeb.Schema.Query.TokenTransfersTest do
"amounts" => Enum.map(token_transfer.amounts, &to_string/1),
"block_number" => token_transfer.block_number,
"log_index" => token_transfer.log_index,
"token_id" => token_transfer.token_id,
"token_ids" => Enum.map(token_transfer.token_ids, &to_string/1),
"from_address_hash" => to_string(token_transfer.from_address_hash),
"to_address_hash" => to_string(token_transfer.to_address_hash),
@ -70,7 +68,6 @@ defmodule BlockScoutWeb.Schema.Query.TokenTransfersTest do
amount
block_number
log_index
token_id
from_address_hash
to_address_hash
token_contract_address_hash

@ -27,7 +27,7 @@ defmodule BlockScoutWeb.Tokens.HelperTest do
test "returns a string with the token_id with ERC-721 token" do
token = build(:token, type: "ERC-721", decimals: nil)
token_transfer = build(:token_transfer, token: token, amount: nil, token_id: 1)
token_transfer = build(:token_transfer, token: token, amount: nil, token_ids: [1])
assert Helper.token_transfer_amount(token_transfer) == {:ok, :erc721_instance}
end

@ -109,8 +109,6 @@ config :explorer, Explorer.Counters.BlockPriorityFeeCounter,
enabled: true,
enable_consolidation: true
config :explorer, Explorer.TokenTransferTokenIdMigration.Supervisor, enabled: true
config :explorer, Explorer.TokenInstanceOwnerAddressMigration.Supervisor, enabled: true
config :explorer, Explorer.Migrator.TransactionsDenormalization, enabled: true

@ -33,8 +33,6 @@ config :explorer, Explorer.Market.History.Cataloger, enabled: false
config :explorer, Explorer.Tracer, disabled?: false
config :explorer, Explorer.TokenTransferTokenIdMigration.Supervisor, enabled: false
config :explorer, Explorer.TokenInstanceOwnerAddressMigration.Supervisor, enabled: false
config :explorer, Explorer.Migrator.TransactionsDenormalization, enabled: false

@ -5,7 +5,7 @@ defmodule Explorer.Application do
use Application
alias Explorer.{Admin, TokenTransferTokenIdMigration}
alias Explorer.Admin
alias Explorer.Chain.Cache.{
Accounts,
@ -122,7 +122,6 @@ defmodule Explorer.Application do
configure(Explorer.Validator.MetadataProcessor),
configure(Explorer.Tags.AddressTag.Cataloger),
configure(MinMissingBlockNumber),
configure(TokenTransferTokenIdMigration.Supervisor),
configure(Explorer.Chain.Fetcher.CheckBytecodeMatchingOnDemand),
configure(Explorer.Chain.Fetcher.FetchValidatorInfoOnDemand),
configure(Explorer.TokenInstanceOwnerAddressMigration.Supervisor),

@ -44,7 +44,6 @@ defmodule Explorer.Chain.TokenTransfer do
* `:to_address_hash` - Address hash foreign key
* `:token_contract_address` - The `t:Explorer.Chain.Address.t/0` of the token's contract.
* `:token_contract_address_hash` - Address hash foreign key
* `:token_id` - ID of the token (applicable to ERC-721 tokens)
* `:transaction` - The `t:Explorer.Chain.Transaction.t/0` ledger
* `:transaction_hash` - Transaction foreign key
* `:log_index` - Index of the corresponding `t:Explorer.Chain.Log.t/0` in the block.
@ -61,7 +60,6 @@ defmodule Explorer.Chain.TokenTransfer do
to_address_hash: Hash.Address.t(),
token_contract_address: %Ecto.Association.NotLoaded{} | Address.t(),
token_contract_address_hash: Hash.Address.t(),
token_id: non_neg_integer() | nil,
transaction: %Ecto.Association.NotLoaded{} | Transaction.t(),
transaction_hash: Hash.Full.t(),
log_index: non_neg_integer(),
@ -86,7 +84,6 @@ defmodule Explorer.Chain.TokenTransfer do
field(:amount, :decimal)
field(:block_number, :integer)
field(:log_index, :integer, primary_key: true)
field(:token_id, :decimal)
field(:amounts, {:array, :decimal})
field(:token_ids, {:array, :decimal})
field(:index_in_batch, :integer, virtual: true)
@ -129,7 +126,7 @@ defmodule Explorer.Chain.TokenTransfer do
end
@required_attrs ~w(block_number log_index from_address_hash to_address_hash token_contract_address_hash transaction_hash block_hash)a
@optional_attrs ~w(amount token_id amounts token_ids)a
@optional_attrs ~w(amount amounts token_ids)a
@doc false
def changeset(%TokenTransfer{} = struct, params \\ %{}) do

@ -120,7 +120,7 @@ defmodule Explorer.Chain.Transaction.StateChange do
end
defp do_update_balance(old_val, type, transfer, _) do
token_ids = if transfer.token.type == "ERC-1155", do: transfer.token_ids || [transfer.token_id], else: [nil]
token_ids = if transfer.token.type == "ERC-1155", do: transfer.token_ids, else: [nil]
transfer_amounts = transfer.amounts || [transfer.amount || 1]
sub_or_add =

@ -1,65 +0,0 @@
defmodule Explorer.TokenTransferTokenIdMigration.LowestBlockNumberUpdater do
@moduledoc """
Collects processed block numbers from token id migration workers
and updates last_processed_block_number according to them.
Full algorithm is in the 'Indexer.Fetcher.TokenTransferTokenIdMigration.Supervisor' module doc.
"""
use GenServer
alias Explorer.Utility.TokenTransferTokenIdMigratorProgress
def start_link(_) do
GenServer.start_link(__MODULE__, :ok, name: __MODULE__)
end
@impl true
def init(_) do
last_processed_block_number = TokenTransferTokenIdMigratorProgress.get_last_processed_block_number()
{:ok, %{last_processed_block_number: last_processed_block_number, processed_ranges: []}}
end
def add_range(from, to) do
GenServer.cast(__MODULE__, {:add_range, from..to})
end
@impl true
def handle_cast({:add_range, range}, %{processed_ranges: processed_ranges} = state) do
ranges =
[range | processed_ranges]
|> Enum.sort_by(& &1.last, &>=/2)
|> normalize_ranges()
{new_last_number, new_ranges} = maybe_update_last_processed_number(state.last_processed_block_number, ranges)
{:noreply, %{last_processed_block_number: new_last_number, processed_ranges: new_ranges}}
end
defp normalize_ranges(ranges) do
%{prev_range: prev, result: result} =
Enum.reduce(ranges, %{prev_range: nil, result: []}, fn range, %{prev_range: prev_range, result: result} ->
case {prev_range, range} do
{nil, _} ->
%{prev_range: range, result: result}
{%{last: l1} = r1, %{first: f2} = r2} when l1 - 1 > f2 ->
%{prev_range: r2, result: [r1 | result]}
{%{first: f1}, %{last: l2}} ->
%{prev_range: f1..l2, result: result}
end
end)
Enum.reverse([prev | result])
end
# since ranges are normalized, we need to check only the first range to determine the new last_processed_number
defp maybe_update_last_processed_number(current_last, [from..to | rest] = ranges) when current_last - 1 <= from do
case TokenTransferTokenIdMigratorProgress.update_last_processed_block_number(to) do
{:ok, _} -> {to, rest}
_ -> {current_last, ranges}
end
end
defp maybe_update_last_processed_number(current_last, ranges), do: {current_last, ranges}
end

@ -1,70 +0,0 @@
defmodule Explorer.TokenTransferTokenIdMigration.Supervisor do
@moduledoc """
Supervises parts of token id migration process.
Migration process algorithm:
Defining the bounds of migration (by the first and the last block number of TokenTransfer).
Supervisor starts the workers in amount equal to 'TOKEN_ID_MIGRATION_CONCURRENCY' env value (defaults to 1)
and the 'LowestBlockNumberUpdater'.
Each worker goes through the token transfers by batches ('TOKEN_ID_MIGRATION_BATCH_SIZE', defaults to 500)
and updates the token_ids to be equal of [token_id] for transfers that has any token_id.
Worker goes from the newest blocks to latest.
After worker is done with current batch, it sends the information about processed batch to 'LowestBlockNumberUpdater'
and takes the next by defining its bounds based on amount of all workers.
For example, if batch size is 10, we have 5 workers and 100 items to be processed,
the distribution will be like this:
1 worker - 99..90, 49..40
2 worker - 89..80, 39..30
3 worker - 79..70, 29..20
4 worker - 69..60, 19..10
5 worker - 59..50, 9..0
'LowestBlockNumberUpdater' keeps the information about the last processed block number
(which is stored in the 'token_transfer_token_id_migrator_progress' db entity)
and block ranges that has already been processed by the workers but couldn't be committed
to last processed block number yet (because of the possible gap between the current last block
and upper bound of the last processed batch). Uncommitted block numbers are stored in normalize ranges.
When there is no gap between the last processed block number and the upper bound of the upper range,
'LowestBlockNumberUpdater' updates the last processed block number in db and drops this range from its state.
This supervisor won't start if the migration is completed
(last processed block number in db == 'TOKEN_ID_MIGRATION_FIRST_BLOCK' (defaults to 0)).
"""
use Supervisor
alias Explorer.TokenTransferTokenIdMigration.{LowestBlockNumberUpdater, Worker}
alias Explorer.Utility.TokenTransferTokenIdMigratorProgress
@default_first_block 0
@default_workers_count 1
def start_link(_) do
Supervisor.start_link(__MODULE__, :ok, name: __MODULE__)
end
@impl true
def init(_) do
first_block = Application.get_env(:explorer, :token_id_migration)[:first_block] || @default_first_block
last_block = TokenTransferTokenIdMigratorProgress.get_last_processed_block_number()
if last_block > first_block do
workers_count = Application.get_env(:explorer, :token_id_migration)[:concurrency] || @default_workers_count
workers =
Enum.map(1..workers_count, fn id ->
Supervisor.child_spec(
{Worker, idx: id, first_block: first_block, last_block: last_block, step: workers_count - 1},
id: {Worker, id},
restart: :transient
)
end)
Supervisor.init([LowestBlockNumberUpdater | workers], strategy: :one_for_one)
else
:ignore
end
end
end

@ -1,84 +0,0 @@
defmodule Explorer.TokenTransferTokenIdMigration.Worker do
@moduledoc """
Performs the migration of TokenTransfer token_id to token_ids by batches.
Full algorithm is in the 'Explorer.TokenTransferTokenIdMigration.Supervisor' module doc.
"""
use GenServer
import Ecto.Query
alias Explorer.Chain.TokenTransfer
alias Explorer.Repo
alias Explorer.TokenTransferTokenIdMigration.LowestBlockNumberUpdater
@default_batch_size 500
@interval 10
def start_link(idx: idx, first_block: first, last_block: last, step: step) do
GenServer.start_link(__MODULE__, %{idx: idx, bottom_block: first, last_block: last, step: step})
end
@impl true
def init(%{idx: idx, bottom_block: bottom_block, last_block: last_block, step: step}) do
batch_size = Application.get_env(:explorer, :token_id_migration)[:batch_size] || @default_batch_size
range = calculate_new_range(last_block, bottom_block, batch_size, idx - 1)
schedule_next_update()
{:ok, %{batch_size: batch_size, bottom_block: bottom_block, step: step, current_range: range}}
end
@impl true
def handle_info(:update, %{current_range: :out_of_bound} = state) do
{:stop, :normal, state}
end
@impl true
def handle_info(:update, %{current_range: {lower_bound, upper_bound}} = state) do
case do_update(lower_bound, upper_bound) do
true ->
LowestBlockNumberUpdater.add_range(upper_bound, lower_bound)
new_range = calculate_new_range(lower_bound, state.bottom_block, state.batch_size, state.step)
schedule_next_update()
{:noreply, %{state | current_range: new_range}}
_ ->
schedule_next_update()
{:noreply, state}
end
end
defp calculate_new_range(last_processed_block, bottom_block, batch_size, step) do
upper_bound = last_processed_block - step * batch_size - 1
lower_bound = max(upper_bound - batch_size + 1, bottom_block)
if upper_bound >= bottom_block do
{lower_bound, upper_bound}
else
:out_of_bound
end
end
defp do_update(lower_bound, upper_bound) do
token_transfers_batch_query =
from(
tt in TokenTransfer,
where: tt.block_number >= ^lower_bound,
where: tt.block_number <= ^upper_bound
)
token_transfers_batch_query
|> Repo.all()
|> Enum.filter(fn %{token_id: token_id} -> not is_nil(token_id) end)
|> Enum.map(fn token_transfer ->
token_transfer
|> TokenTransfer.changeset(%{token_ids: [token_transfer.token_id], token_id: nil})
|> Repo.update()
end)
|> Enum.all?(&match?({:ok, _}, &1))
end
defp schedule_next_update do
Process.send_after(self(), :update, @interval)
end
end

@ -1,67 +0,0 @@
defmodule Explorer.Utility.TokenTransferTokenIdMigratorProgress do
@moduledoc """
Module is responsible for keeping the current progress of TokenTransfer token_id migration.
Full algorithm is in the 'Indexer.Fetcher.TokenTransferTokenIdMigration.Supervisor' module doc.
"""
use Explorer.Schema
require Logger
alias Explorer.Chain.Cache.BlockNumber
alias Explorer.Repo
schema "token_transfer_token_id_migrator_progress" do
field(:last_processed_block_number, :integer)
timestamps()
end
@doc false
def changeset(progress \\ %__MODULE__{}, params) do
cast(progress, params, [:last_processed_block_number])
end
def get_current_progress do
Repo.one(
from(
p in __MODULE__,
order_by: [desc: p.updated_at],
limit: 1
)
)
end
def get_last_processed_block_number do
case get_current_progress() do
nil ->
latest_processed_block_number = BlockNumber.get_max() + 1
update_last_processed_block_number(latest_processed_block_number)
latest_processed_block_number
%{last_processed_block_number: block_number} ->
block_number
end
end
def update_last_processed_block_number(block_number, force \\ false) do
case get_current_progress() do
nil ->
%{last_processed_block_number: block_number}
|> changeset()
|> Repo.insert()
progress ->
if not force and progress.last_processed_block_number < block_number do
Logger.error(
"TokenTransferTokenIdMigratorProgress new block_number is above the last one. Last: #{progress.last_processed_block_number}, new: #{block_number}"
)
{:error, :invalid_block_number}
else
progress
|> changeset(%{last_processed_block_number: block_number})
|> Repo.update()
end
end
end
end

@ -0,0 +1,12 @@
defmodule Explorer.Repo.Migrations.DropTokenTransfersTokenIdColumn do
use Ecto.Migration
def change do
drop(index(:token_transfers, [:token_id]))
drop(index(:token_transfers, [:token_contract_address_hash, "token_id DESC", "block_number DESC"]))
alter table(:token_transfers) do
remove(:token_id)
end
end
end

@ -1,37 +0,0 @@
defmodule Explorer.TokenTransferTokenIdMigration.LowestBlockNumberUpdaterTest do
use Explorer.DataCase, async: false
alias Explorer.Repo
alias Explorer.TokenTransferTokenIdMigration.LowestBlockNumberUpdater
alias Explorer.Utility.TokenTransferTokenIdMigratorProgress
describe "Add range and update last processed block number" do
test "add_range/2" do
TokenTransferTokenIdMigratorProgress.update_last_processed_block_number(2000, true)
LowestBlockNumberUpdater.start_link([])
LowestBlockNumberUpdater.add_range(1000, 500)
LowestBlockNumberUpdater.add_range(1500, 1001)
Process.sleep(10)
assert %{last_processed_block_number: 2000, processed_ranges: [1500..500//-1]} =
:sys.get_state(LowestBlockNumberUpdater)
assert %{last_processed_block_number: 2000} = Repo.one(TokenTransferTokenIdMigratorProgress)
LowestBlockNumberUpdater.add_range(499, 300)
LowestBlockNumberUpdater.add_range(299, 0)
Process.sleep(10)
assert %{last_processed_block_number: 2000, processed_ranges: [1500..0//-1]} =
:sys.get_state(LowestBlockNumberUpdater)
assert %{last_processed_block_number: 2000} = Repo.one(TokenTransferTokenIdMigratorProgress)
LowestBlockNumberUpdater.add_range(1999, 1501)
Process.sleep(10)
assert %{last_processed_block_number: 0, processed_ranges: []} = :sys.get_state(LowestBlockNumberUpdater)
assert %{last_processed_block_number: 0} = Repo.one(TokenTransferTokenIdMigratorProgress)
end
end
end

@ -1,31 +0,0 @@
defmodule Explorer.TokenTransferTokenIdMigration.WorkerTest do
use Explorer.DataCase, async: false
alias Explorer.Repo
alias Explorer.TokenTransferTokenIdMigration.{LowestBlockNumberUpdater, Worker}
alias Explorer.Utility.TokenTransferTokenIdMigratorProgress
describe "Move TokenTransfer token_id to token_ids" do
test "Move token_ids and update last processed block number" do
insert(:token_transfer, block_number: 1, token_id: 1, transaction: insert(:transaction))
insert(:token_transfer, block_number: 500, token_id: 2, transaction: insert(:transaction))
insert(:token_transfer, block_number: 1000, token_id: 3, transaction: insert(:transaction))
insert(:token_transfer, block_number: 1500, token_id: 4, transaction: insert(:transaction))
insert(:token_transfer, block_number: 2000, token_id: 5, transaction: insert(:transaction))
TokenTransferTokenIdMigratorProgress.update_last_processed_block_number(3000, true)
LowestBlockNumberUpdater.start_link([])
Worker.start_link(idx: 1, first_block: 0, last_block: 3000, step: 2)
Worker.start_link(idx: 2, first_block: 0, last_block: 3000, step: 2)
Worker.start_link(idx: 3, first_block: 0, last_block: 3000, step: 2)
Process.sleep(200)
token_transfers = Repo.all(Explorer.Chain.TokenTransfer)
assert Enum.all?(token_transfers, fn tt -> is_nil(tt.token_id) end)
expected_token_ids = [[Decimal.new(1)], [Decimal.new(2)], [Decimal.new(3)], [Decimal.new(4)], [Decimal.new(5)]]
assert ^expected_token_ids = token_transfers |> Enum.map(& &1.token_ids) |> Enum.sort_by(&List.first/1)
end
end
end
Loading…
Cancel
Save