Create Indexer.Fetcher.TokenInstance.{SanitizeERC721, SanitizeERC1155… (#9226)
* Create Indexer.Fetcher.TokenInstance.{SanitizeERC721, SanitizeERC1155}; Move token instances to BlockReferencing stage
* Add envs to .env file
* Fix dialyzer
* Process review comments
* Add env to .env file
vb-metadata-from-base-uri
nikitosing
9 months ago
committed by
GitHub
parent
ebfc315838
commit
389debbc85
@ -0,0 +1,51 @@ |
||||
defmodule Indexer.Fetcher.TokenInstance.SanitizeERC1155 do |
||||
@moduledoc """ |
||||
This fetcher is stands for creating token instances which wasn't inserted yet and index meta for them. |
||||
|
||||
!!!Imports only ERC-1155 token instances!!! |
||||
""" |
||||
|
||||
use GenServer, restart: :transient |
||||
|
||||
alias Explorer.Chain.Token.Instance |
||||
alias Explorer.Repo |
||||
|
||||
import Indexer.Fetcher.TokenInstance.Helper |
||||
|
||||
def start_link(_) do |
||||
concurrency = Application.get_env(:indexer, __MODULE__)[:concurrency] |
||||
batch_size = Application.get_env(:indexer, __MODULE__)[:batch_size] |
||||
GenServer.start_link(__MODULE__, %{concurrency: concurrency, batch_size: batch_size}, name: __MODULE__) |
||||
end |
||||
|
||||
@impl true |
||||
def init(opts) do |
||||
GenServer.cast(__MODULE__, :backfill) |
||||
|
||||
{:ok, opts} |
||||
end |
||||
|
||||
@impl true |
||||
def handle_cast(:backfill, %{concurrency: concurrency, batch_size: batch_size} = state) do |
||||
instances_to_fetch = |
||||
(concurrency * batch_size) |
||||
|> Instance.not_inserted_erc_1155_token_instances() |
||||
|> Repo.all() |
||||
|
||||
if Enum.empty?(instances_to_fetch) do |
||||
{:stop, :normal, state} |
||||
else |
||||
instances_to_fetch |
||||
|> Enum.uniq() |
||||
|> Enum.chunk_every(batch_size) |
||||
|> Enum.map(&process_batch/1) |
||||
|> Task.await_many(:infinity) |
||||
|
||||
GenServer.cast(__MODULE__, :backfill) |
||||
|
||||
{:noreply, state} |
||||
end |
||||
end |
||||
|
||||
defp process_batch(batch), do: Task.async(fn -> batch_fetch_instances(batch) end) |
||||
end |
||||
@ -0,0 +1,89 @@ |
||||
defmodule Indexer.Fetcher.TokenInstance.SanitizeERC721 do |
||||
@moduledoc """ |
||||
This fetcher is stands for creating token instances which wasn't inserted yet and index meta for them. |
||||
|
||||
!!!Imports only ERC-721 token instances!!! |
||||
""" |
||||
|
||||
use GenServer, restart: :transient |
||||
|
||||
alias Explorer.Application.Constants |
||||
alias Explorer.Chain.Token |
||||
alias Explorer.Chain.Token.Instance |
||||
alias Explorer.Repo |
||||
|
||||
import Indexer.Fetcher.TokenInstance.Helper |
||||
|
||||
def start_link(_) do |
||||
concurrency = Application.get_env(:indexer, __MODULE__)[:concurrency] |
||||
batch_size = Application.get_env(:indexer, __MODULE__)[:batch_size] |
||||
tokens_queue_size = Application.get_env(:indexer, __MODULE__)[:tokens_queue_size] |
||||
|
||||
GenServer.start_link( |
||||
__MODULE__, |
||||
%{concurrency: concurrency, batch_size: batch_size, tokens_queue_size: tokens_queue_size}, |
||||
name: __MODULE__ |
||||
) |
||||
end |
||||
|
||||
@impl true |
||||
def init(opts) do |
||||
last_token_address_hash = Constants.get_last_processed_token_address_hash() |
||||
GenServer.cast(__MODULE__, :fetch_tokens_queue) |
||||
|
||||
{:ok, Map.put(opts, :last_token_address_hash, last_token_address_hash)} |
||||
end |
||||
|
||||
@impl true |
||||
def handle_cast(:fetch_tokens_queue, state) do |
||||
address_hashes = |
||||
state[:tokens_queue_size] |
||||
|> Token.ordered_erc_721_token_address_hashes_list_query(state[:last_token_address_hash]) |
||||
|> Repo.all() |
||||
|
||||
if Enum.empty?(address_hashes) do |
||||
{:stop, :normal, state} |
||||
else |
||||
GenServer.cast(__MODULE__, :backfill) |
||||
|
||||
{:noreply, Map.put(state, :tokens_queue, address_hashes)} |
||||
end |
||||
end |
||||
|
||||
@impl true |
||||
def handle_cast(:backfill, %{tokens_queue: []} = state) do |
||||
GenServer.cast(__MODULE__, :fetch_tokens_queue) |
||||
|
||||
{:noreply, state} |
||||
end |
||||
|
||||
@impl true |
||||
def handle_cast( |
||||
:backfill, |
||||
%{concurrency: concurrency, batch_size: batch_size, tokens_queue: [current_address_hash | remains]} = state |
||||
) do |
||||
instances_to_fetch = |
||||
(concurrency * batch_size) |
||||
|> Instance.not_inserted_token_instances_query_by_token(current_address_hash) |
||||
|> Repo.all() |
||||
|
||||
if Enum.empty?(instances_to_fetch) do |
||||
Constants.insert_last_processed_token_address_hash(current_address_hash) |
||||
GenServer.cast(__MODULE__, :backfill) |
||||
|
||||
{:noreply, %{state | tokens_queue: remains, last_token_address_hash: current_address_hash}} |
||||
else |
||||
instances_to_fetch |
||||
|> Enum.uniq() |
||||
|> Enum.chunk_every(batch_size) |
||||
|> Enum.map(&process_batch/1) |
||||
|> Task.await_many(:infinity) |
||||
|
||||
GenServer.cast(__MODULE__, :backfill) |
||||
|
||||
{:noreply, state} |
||||
end |
||||
end |
||||
|
||||
defp process_batch(batch), do: Task.async(fn -> batch_fetch_instances(batch) end) |
||||
end |
||||
@ -0,0 +1,33 @@ |
||||
defmodule Indexer.Fetcher.TokenInstance.SanitizeERC1155Test do |
||||
use Explorer.DataCase |
||||
|
||||
alias Explorer.Repo |
||||
alias Explorer.Chain.Token.Instance |
||||
alias EthereumJSONRPC.Encoder |
||||
|
||||
describe "sanitizer test" do |
||||
test "imports token instances" do |
||||
for i <- 0..3 do |
||||
token = insert(:token, type: "ERC-1155") |
||||
|
||||
insert(:address_current_token_balance, |
||||
token_type: "ERC-1155", |
||||
token_id: i, |
||||
token_contract_address_hash: token.contract_address_hash, |
||||
value: Enum.random(1..100_000) |
||||
) |
||||
end |
||||
|
||||
assert [] = Repo.all(Instance) |
||||
|
||||
start_supervised!({Indexer.Fetcher.TokenInstance.SanitizeERC1155, []}) |
||||
|
||||
:timer.sleep(500) |
||||
|
||||
instances = Repo.all(Instance) |
||||
|
||||
assert Enum.count(instances) == 4 |
||||
assert Enum.all?(instances, fn instance -> !is_nil(instance.error) and is_nil(instance.metadata) end) |
||||
end |
||||
end |
||||
end |
||||
@ -0,0 +1,39 @@ |
||||
defmodule Indexer.Fetcher.TokenInstance.SanitizeERC721Test do |
||||
use Explorer.DataCase |
||||
|
||||
alias Explorer.Repo |
||||
alias Explorer.Chain.Token.Instance |
||||
alias EthereumJSONRPC.Encoder |
||||
|
||||
describe "sanitizer test" do |
||||
test "imports token instances" do |
||||
for x <- 0..3 do |
||||
erc_721_token = insert(:token, type: "ERC-721") |
||||
|
||||
tx = insert(:transaction, input: "0xabcd010203040506") |> with_block() |
||||
|
||||
address = insert(:address) |
||||
|
||||
insert(:token_transfer, |
||||
transaction: tx, |
||||
block: tx.block, |
||||
block_number: tx.block_number, |
||||
from_address: address, |
||||
token_contract_address: erc_721_token.contract_address, |
||||
token_ids: [x] |
||||
) |
||||
end |
||||
|
||||
assert [] = Repo.all(Instance) |
||||
|
||||
start_supervised!({Indexer.Fetcher.TokenInstance.SanitizeERC721, []}) |
||||
|
||||
:timer.sleep(500) |
||||
|
||||
instances = Repo.all(Instance) |
||||
|
||||
assert Enum.count(instances) == 4 |
||||
assert Enum.all?(instances, fn instance -> !is_nil(instance.error) and is_nil(instance.metadata) end) |
||||
end |
||||
end |
||||
end |
||||
Loading…
Reference in new issue