feat: API endpoint to re-fetch token instance metadata (#10097)
* feat: Re-fetch token instance metadata
* Partially process review comments
* Process reviewer comments. Part 2
* Process reviewer comments. Part 3
* Process reviewer comments. Part 4
* Fix events
* Add test
* Remove :token preload
* fix formatting
* Fix tests
* Remove unused aliases
* Add reCAPTCHA for token instance re-fetch API endpoint
* Check event on websocket at /api/v2/tokens/{address_hash}/instances/{token_id}/refetch-metadata endpoint
pull/10240/head
Victor Baranov
5 months ago
committed by
GitHub
parent
c31f937680
commit
4297704b8e
@ -0,0 +1,26 @@ |
||||
defmodule BlockScoutWeb.TokenInstanceChannel do |
||||
@moduledoc """ |
||||
Establishes pub/sub channel for live updates of token instances events. |
||||
""" |
||||
use BlockScoutWeb, :channel |
||||
|
||||
intercept(["fetched_token_instance_metadata"]) |
||||
|
||||
def join("fetched_token_instance_metadata", _params, socket) do |
||||
{:ok, %{}, socket} |
||||
end |
||||
|
||||
def join("token_instances:" <> _token_contract_address_hash, _params, socket) do |
||||
{:ok, %{}, socket} |
||||
end |
||||
|
||||
def handle_out( |
||||
"fetched_token_instance_metadata", |
||||
res, |
||||
%Phoenix.Socket{handler: BlockScoutWeb.UserSocketV2} = socket |
||||
) do |
||||
push(socket, "fetched_token_instance_metadata", res) |
||||
|
||||
{:noreply, socket} |
||||
end |
||||
end |
||||
@ -1,4 +1,4 @@ |
||||
defmodule BlockScoutWeb.AdminRouter do |
||||
defmodule BlockScoutWeb.Routers.AdminRouter do |
||||
@moduledoc """ |
||||
Router for admin pages. |
||||
""" |
||||
@ -1,4 +1,4 @@ |
||||
defmodule BlockScoutWeb.APIKeyV2Router do |
||||
defmodule BlockScoutWeb.Routers.APIKeyV2Router do |
||||
@moduledoc """ |
||||
Router for /api/v2/key. This route has separate router in order to avoid rate limiting |
||||
""" |
||||
@ -0,0 +1,71 @@ |
||||
# This file in ignore list of `sobelow`, be careful while adding new endpoints here |
||||
defmodule BlockScoutWeb.Routers.TokensApiV2Router do |
||||
@moduledoc """ |
||||
Router for /api/v2/tokens. This route has separate router in order to ignore sobelow's warning about missing CSRF protection |
||||
""" |
||||
use BlockScoutWeb, :router |
||||
alias BlockScoutWeb.API.V2 |
||||
alias BlockScoutWeb.Plug.{CheckApiV2, RateLimit} |
||||
|
||||
@max_query_string_length 5_000 |
||||
|
||||
pipeline :api_v2 do |
||||
plug( |
||||
Plug.Parsers, |
||||
parsers: [:urlencoded, :multipart, :json], |
||||
query_string_length: @max_query_string_length, |
||||
pass: ["*/*"], |
||||
json_decoder: Poison |
||||
) |
||||
|
||||
plug(BlockScoutWeb.Plug.Logger, application: :api_v2) |
||||
plug(:accepts, ["json"]) |
||||
plug(CheckApiV2) |
||||
plug(:fetch_session) |
||||
plug(:protect_from_forgery) |
||||
plug(RateLimit) |
||||
end |
||||
|
||||
pipeline :api_v2_no_forgery_protect do |
||||
plug( |
||||
Plug.Parsers, |
||||
parsers: [:urlencoded, :multipart, :json], |
||||
length: 20_000_000, |
||||
query_string_length: 5_000, |
||||
pass: ["*/*"], |
||||
json_decoder: Poison |
||||
) |
||||
|
||||
plug(BlockScoutWeb.Plug.Logger, application: :api_v2) |
||||
plug(:accepts, ["json"]) |
||||
plug(CheckApiV2) |
||||
plug(RateLimit) |
||||
plug(:fetch_session) |
||||
end |
||||
|
||||
scope "/", as: :api_v2 do |
||||
pipe_through(:api_v2_no_forgery_protect) |
||||
|
||||
patch("/:address_hash_param/instances/:token_id/refetch-metadata", V2.TokenController, :refetch_metadata) |
||||
end |
||||
|
||||
scope "/", as: :api_v2 do |
||||
pipe_through(:api_v2) |
||||
|
||||
if Application.compile_env(:explorer, Explorer.Chain.BridgedToken)[:enabled] do |
||||
get("/bridged", V2.TokenController, :bridged_tokens_list) |
||||
end |
||||
|
||||
get("/", V2.TokenController, :tokens_list) |
||||
get("/:address_hash_param", V2.TokenController, :token) |
||||
get("/:address_hash_param/counters", V2.TokenController, :counters) |
||||
get("/:address_hash_param/transfers", V2.TokenController, :transfers) |
||||
get("/:address_hash_param/holders", V2.TokenController, :holders) |
||||
get("/:address_hash_param/holders/csv", V2.CSVExportController, :export_token_holders) |
||||
get("/:address_hash_param/instances", V2.TokenController, :instances) |
||||
get("/:address_hash_param/instances/:token_id", V2.TokenController, :instance) |
||||
get("/:address_hash_param/instances/:token_id/transfers", V2.TokenController, :transfers_by_instance) |
||||
get("/:address_hash_param/instances/:token_id/holders", V2.TokenController, :holders_by_instance) |
||||
get("/:address_hash_param/instances/:token_id/transfers-count", V2.TokenController, :transfers_count_by_instance) |
||||
end |
||||
end |
||||
@ -1,5 +1,5 @@ |
||||
# This file in ignore list of `sobelow`, be careful while adding new endpoints here |
||||
defmodule BlockScoutWeb.UtilsApiV2Router do |
||||
defmodule BlockScoutWeb.Routers.UtilsApiV2Router do |
||||
@moduledoc """ |
||||
Router for /api/v2/utils. This route has separate router in order to ignore sobelow's warning about missing CSRF protection |
||||
""" |
||||
@ -1,4 +1,4 @@ |
||||
defmodule BlockScoutWeb.WebRouter do |
||||
defmodule BlockScoutWeb.Routers.WebRouter do |
||||
@moduledoc """ |
||||
Router for web app |
||||
""" |
||||
@ -1,7 +1,7 @@ |
||||
defmodule BlockScoutWeb.Admin.SessionView do |
||||
use BlockScoutWeb, :view |
||||
|
||||
import BlockScoutWeb.AdminRouter.Helpers |
||||
import BlockScoutWeb.Routers.AdminRouter.Helpers |
||||
|
||||
alias BlockScoutWeb.FormView |
||||
end |
||||
|
||||
@ -1,7 +1,7 @@ |
||||
defmodule BlockScoutWeb.Admin.SetupView do |
||||
use BlockScoutWeb, :view |
||||
|
||||
import BlockScoutWeb.AdminRouter.Helpers |
||||
import BlockScoutWeb.Routers.AdminRouter.Helpers |
||||
|
||||
alias BlockScoutWeb.FormView |
||||
end |
||||
|
||||
@ -0,0 +1,87 @@ |
||||
defmodule Explorer.Utility.TokenInstanceMetadataRefetchAttempt do |
||||
@moduledoc """ |
||||
Module is responsible for keeping the number of retries for |
||||
Indexer.Fetcher.OnDemand.TokenInstanceMetadataRefetch. |
||||
""" |
||||
|
||||
use Explorer.Schema |
||||
|
||||
alias Explorer.Chain.Hash |
||||
alias Explorer.Repo |
||||
|
||||
@primary_key false |
||||
typed_schema "token_instance_metadata_refetch_attempts" do |
||||
field(:token_contract_address_hash, Hash.Address, primary_key: true) |
||||
field(:token_id, :decimal, primary_key: true) |
||||
field(:retries_number, :integer, primary_key: false) |
||||
|
||||
timestamps() |
||||
end |
||||
|
||||
@doc false |
||||
def changeset(token_instance_metadata_refetch_attempt \\ %__MODULE__{}, params) do |
||||
cast(token_instance_metadata_refetch_attempt, params, [:hash, :retries_number]) |
||||
end |
||||
|
||||
@doc """ |
||||
Gets retries number and updated_at for given token contract Explorer.Chain.Address and token_id |
||||
""" |
||||
@spec get_retries_number(Hash.Address.t(), non_neg_integer()) :: {non_neg_integer(), DateTime.t()} | nil |
||||
def get_retries_number(token_contract_address_hash, token_id) do |
||||
__MODULE__ |
||||
|> where( |
||||
[token_instance_metadata_refetch_attempt], |
||||
token_instance_metadata_refetch_attempt.token_contract_address_hash == ^token_contract_address_hash |
||||
) |
||||
|> where([token_instance_metadata_refetch_attempt], token_instance_metadata_refetch_attempt.token_id == ^token_id) |
||||
|> select( |
||||
[token_instance_metadata_refetch_attempt], |
||||
{token_instance_metadata_refetch_attempt.retries_number, token_instance_metadata_refetch_attempt.updated_at} |
||||
) |
||||
|> Repo.one() |
||||
end |
||||
|
||||
@doc """ |
||||
Inserts the number of retries for fetching token instance metadata into the database. |
||||
|
||||
## Parameters |
||||
- `token_contract_address_hash` - The hash of the token contract address. |
||||
- `token_id` - The ID of the token instance. |
||||
|
||||
## Returns |
||||
The result of the insertion operation. |
||||
|
||||
""" |
||||
@spec insert_retries_number(Hash.Address.t(), non_neg_integer()) :: {non_neg_integer(), nil | [term()]} |
||||
def insert_retries_number(token_contract_address_hash, token_id) do |
||||
now = DateTime.utc_now() |
||||
|
||||
params = [ |
||||
%{ |
||||
token_contract_address_hash: token_contract_address_hash, |
||||
token_id: token_id, |
||||
inserted_at: now, |
||||
updated_at: now, |
||||
retries_number: 1 |
||||
} |
||||
] |
||||
|
||||
Repo.insert_all(__MODULE__, params, |
||||
on_conflict: default_on_conflict(), |
||||
conflict_target: [:token_contract_address_hash, :token_id] |
||||
) |
||||
end |
||||
|
||||
defp default_on_conflict do |
||||
from( |
||||
token_instance_metadata_refetch_attempt in __MODULE__, |
||||
update: [ |
||||
set: [ |
||||
retries_number: fragment("? + 1", token_instance_metadata_refetch_attempt.retries_number), |
||||
inserted_at: fragment("LEAST(?, EXCLUDED.inserted_at)", token_instance_metadata_refetch_attempt.inserted_at), |
||||
updated_at: fragment("GREATEST(?, EXCLUDED.updated_at)", token_instance_metadata_refetch_attempt.updated_at) |
||||
] |
||||
] |
||||
) |
||||
end |
||||
end |
||||
@ -0,0 +1,15 @@ |
||||
defmodule Explorer.Repo.Migrations.CreateTokenInstanceMetadataRefetchAttemptsTable do |
||||
use Ecto.Migration |
||||
|
||||
def change do |
||||
create table(:token_instance_metadata_refetch_attempts, primary_key: false) do |
||||
add(:token_contract_address_hash, :bytea, null: false, primary_key: true) |
||||
add(:token_id, :numeric, precision: 78, scale: 0, null: false, primary_key: true) |
||||
add(:retries_number, :smallint) |
||||
|
||||
timestamps() |
||||
end |
||||
|
||||
create(index(:token_instance_metadata_refetch_attempts, [:token_contract_address_hash, :token_id])) |
||||
end |
||||
end |
||||
@ -0,0 +1,123 @@ |
||||
defmodule Indexer.Fetcher.OnDemand.TokenInstanceMetadataRefetch do |
||||
@moduledoc """ |
||||
Re-fetches token instance metadata. |
||||
""" |
||||
|
||||
require Logger |
||||
|
||||
use GenServer |
||||
use Indexer.Fetcher, restart: :permanent |
||||
|
||||
alias Explorer.Chain.Events.Publisher |
||||
alias Explorer.Chain.Token.Instance, as: TokenInstance |
||||
alias Explorer.Counters.Helper, as: CountersHelper |
||||
alias Explorer.SmartContract.Reader |
||||
alias Explorer.Token.MetadataRetriever |
||||
alias Explorer.Utility.TokenInstanceMetadataRefetchAttempt |
||||
alias Indexer.Fetcher.TokenInstance.Helper, as: TokenInstanceHelper |
||||
|
||||
@max_delay :timer.hours(168) |
||||
|
||||
@spec trigger_refetch(TokenInstance.t()) :: :ok |
||||
def trigger_refetch(token_instance) do |
||||
unless is_nil(token_instance.metadata) do |
||||
GenServer.cast(__MODULE__, {:refetch, token_instance}) |
||||
end |
||||
end |
||||
|
||||
defp fetch_metadata(token_instance, state) do |
||||
with {:retries_number, {retries_number, updated_at}} <- |
||||
{:retries_number, |
||||
TokenInstanceMetadataRefetchAttempt.get_retries_number( |
||||
token_instance.token_contract_address_hash, |
||||
token_instance.token_id |
||||
)}, |
||||
updated_at_ms = DateTime.to_unix(updated_at, :millisecond), |
||||
{:retry, true} <- |
||||
{:retry, |
||||
CountersHelper.current_time() - updated_at_ms > |
||||
threshold(retries_number)} do |
||||
fetch_and_broadcast_metadata(token_instance, state) |
||||
else |
||||
{:retries_number, nil} -> |
||||
fetch_and_broadcast_metadata(token_instance, state) |
||||
|
||||
{:retry, false} -> |
||||
:ok |
||||
end |
||||
end |
||||
|
||||
defp fetch_and_broadcast_metadata(token_instance, _state) do |
||||
from_base_uri? = Application.get_env(:indexer, __MODULE__)[:base_uri_retry?] |
||||
|
||||
token_id = TokenInstanceHelper.prepare_token_id(token_instance.token_id) |
||||
contract_address_hash_string = to_string(token_instance.token_contract_address_hash) |
||||
|
||||
request = |
||||
TokenInstanceHelper.prepare_request( |
||||
token_instance.token.type, |
||||
contract_address_hash_string, |
||||
token_id, |
||||
false |
||||
) |
||||
|
||||
result = |
||||
case Reader.query_contracts([request], TokenInstanceHelper.erc_721_1155_abi(), [], false) do |
||||
[ok: [uri]] -> |
||||
{:ok, [uri]} |
||||
|
||||
_ -> |
||||
nil |
||||
end |
||||
|
||||
with {:empty_result, false} <- {:empty_result, is_nil(result)}, |
||||
{:fetched_metadata, {:ok, %{metadata: metadata}}} <- |
||||
{:fetched_metadata, MetadataRetriever.fetch_json(result, token_id, nil, from_base_uri?)} do |
||||
TokenInstance.set_metadata(token_instance, metadata) |
||||
|
||||
Publisher.broadcast( |
||||
%{fetched_token_instance_metadata: [to_string(token_instance.token_contract_address_hash), token_id, metadata]}, |
||||
:on_demand |
||||
) |
||||
else |
||||
{:empty_result, true} -> |
||||
:ok |
||||
|
||||
{:fetched_metadata, _error} -> |
||||
Logger.error(fn -> |
||||
"Error while setting address #{inspect(to_string(token_instance.token_contract_address_hash))} metadata" |
||||
end) |
||||
|
||||
TokenInstanceMetadataRefetchAttempt.insert_retries_number( |
||||
token_instance.token_contract_address_hash, |
||||
token_instance.token_id |
||||
) |
||||
end |
||||
end |
||||
|
||||
def start_link([init_opts, server_opts]) do |
||||
GenServer.start_link(__MODULE__, init_opts, server_opts) |
||||
end |
||||
|
||||
@impl true |
||||
def init(json_rpc_named_arguments) do |
||||
{:ok, %{json_rpc_named_arguments: json_rpc_named_arguments}} |
||||
end |
||||
|
||||
@impl true |
||||
def handle_cast({:refetch, token_instance}, state) do |
||||
fetch_metadata(token_instance, state) |
||||
|
||||
{:noreply, state} |
||||
end |
||||
|
||||
defp update_threshold_ms do |
||||
Application.get_env(:indexer, __MODULE__)[:threshold] |
||||
end |
||||
|
||||
defp threshold(retries_number) do |
||||
delay_in_ms = trunc(update_threshold_ms() * :math.pow(2, retries_number)) |
||||
|
||||
min(delay_in_ms, @max_delay) |
||||
end |
||||
end |
||||
@ -0,0 +1,181 @@ |
||||
defmodule Indexer.Fetcher.OnDemand.TokenInstanceMetadataRefetchTest do |
||||
use EthereumJSONRPC.Case, async: false |
||||
use Explorer.DataCase |
||||
|
||||
import Mox |
||||
|
||||
alias Explorer.Chain.Token.Instance, as: TokenInstance |
||||
alias Explorer.Chain.Events.Subscriber |
||||
alias Explorer.TestHelper |
||||
alias Explorer.Utility.TokenInstanceMetadataRefetchAttempt |
||||
alias Indexer.Fetcher.OnDemand.TokenInstanceMetadataRefetch, as: TokenInstanceMetadataRefetchOnDemand |
||||
|
||||
@moduletag :capture_log |
||||
|
||||
setup :set_mox_global |
||||
|
||||
setup :verify_on_exit! |
||||
|
||||
setup %{json_rpc_named_arguments: json_rpc_named_arguments} do |
||||
mocked_json_rpc_named_arguments = Keyword.put(json_rpc_named_arguments, :transport, EthereumJSONRPC.Mox) |
||||
|
||||
start_supervised!({Task.Supervisor, name: Indexer.TaskSupervisor}) |
||||
|
||||
start_supervised!( |
||||
{TokenInstanceMetadataRefetchOnDemand, |
||||
[mocked_json_rpc_named_arguments, [name: TokenInstanceMetadataRefetchOnDemand]]} |
||||
) |
||||
|
||||
%{json_rpc_named_arguments: mocked_json_rpc_named_arguments} |
||||
end |
||||
|
||||
describe "refetch token instance metadata behaviour" do |
||||
setup do |
||||
Subscriber.to(:fetched_token_instance_metadata, :on_demand) |
||||
|
||||
:ok |
||||
end |
||||
|
||||
test "token instance broadcasts fetched token instance metadata" do |
||||
token = insert(:token, name: "Super Token", type: "ERC-721") |
||||
token_id = 1 |
||||
|
||||
token_instance = |
||||
insert(:token_instance, |
||||
token_id: token_id, |
||||
token_contract_address_hash: token.contract_address_hash, |
||||
metadata: %{} |
||||
) |
||||
|> Repo.preload(:token) |
||||
|
||||
metadata = %{"name" => "Super Token"} |
||||
url = "http://metadata.endpoint.com" |
||||
token_contract_address_hash_string = to_string(token.contract_address_hash) |
||||
|
||||
TestHelper.fetch_token_uri_mock(url, token_contract_address_hash_string) |
||||
|
||||
Application.put_env(:explorer, :http_adapter, Explorer.Mox.HTTPoison) |
||||
|
||||
Explorer.Mox.HTTPoison |
||||
|> expect(:get, fn ^url, _headers, _options -> |
||||
{:ok, %HTTPoison.Response{status_code: 200, body: Jason.encode!(metadata)}} |
||||
end) |
||||
|
||||
assert TokenInstanceMetadataRefetchOnDemand.trigger_refetch(token_instance) == :ok |
||||
|
||||
:timer.sleep(100) |
||||
|
||||
token_instance_from_db = |
||||
Repo.get_by(TokenInstance, token_id: token_id, token_contract_address_hash: token.contract_address_hash) |
||||
|
||||
assert(token_instance_from_db) |
||||
refute is_nil(token_instance_from_db.metadata) |
||||
assert token_instance_from_db.metadata == metadata |
||||
|
||||
assert is_nil( |
||||
Repo.get_by(TokenInstanceMetadataRefetchAttempt, |
||||
token_contract_address_hash: token.contract_address_hash, |
||||
token_id: token_id |
||||
) |
||||
) |
||||
|
||||
assert_receive( |
||||
{:chain_event, :fetched_token_instance_metadata, :on_demand, |
||||
[^token_contract_address_hash_string, ^token_id, ^metadata]} |
||||
) |
||||
|
||||
Application.put_env(:explorer, :http_adapter, HTTPoison) |
||||
end |
||||
|
||||
test "don't run the update on the token instance with no metadata fetched initially" do |
||||
token = insert(:token, name: "Super Token", type: "ERC-721") |
||||
token_id = 1 |
||||
|
||||
token_instance = |
||||
insert(:token_instance, |
||||
token_id: token_id, |
||||
token_contract_address_hash: token.contract_address_hash, |
||||
metadata: nil |
||||
) |
||||
|> Repo.preload(:token) |
||||
|
||||
metadata = %{"name" => "Super Token"} |
||||
token_contract_address_hash_string = to_string(token.contract_address_hash) |
||||
|
||||
assert TokenInstanceMetadataRefetchOnDemand.trigger_refetch(token_instance) == nil |
||||
|
||||
:timer.sleep(100) |
||||
|
||||
token_instance_from_db = |
||||
Repo.get_by(TokenInstance, token_id: token_id, token_contract_address_hash: token.contract_address_hash) |
||||
|
||||
assert(token_instance_from_db) |
||||
assert is_nil(token_instance_from_db.metadata) |
||||
|
||||
assert is_nil( |
||||
Repo.get_by(TokenInstanceMetadataRefetchAttempt, |
||||
token_contract_address_hash: token.contract_address_hash, |
||||
token_id: token_id |
||||
) |
||||
) |
||||
|
||||
refute_receive( |
||||
{:chain_event, :fetched_token_instance_metadata, :on_demand, |
||||
[^token_contract_address_hash_string, ^token_id, %{metadata: ^metadata}]} |
||||
) |
||||
end |
||||
|
||||
test "updates token_instance_metadata_refetch_attempts table" do |
||||
token = insert(:token, name: "Super Token", type: "ERC-721") |
||||
token_id = 1 |
||||
|
||||
token_instance = |
||||
insert(:token_instance, |
||||
token_id: token_id, |
||||
token_contract_address_hash: token.contract_address_hash, |
||||
metadata: %{} |
||||
) |
||||
|> Repo.preload(:token) |
||||
|
||||
metadata = %{"name" => "Super Token"} |
||||
url = "http://metadata.endpoint.com" |
||||
token_contract_address_hash_string = to_string(token.contract_address_hash) |
||||
|
||||
TestHelper.fetch_token_uri_mock(url, token_contract_address_hash_string) |
||||
|
||||
Application.put_env(:explorer, :http_adapter, Explorer.Mox.HTTPoison) |
||||
|
||||
Explorer.Mox.HTTPoison |
||||
|> expect(:get, fn ^url, _headers, _options -> |
||||
{:ok, %HTTPoison.Response{status_code: 200, body: nil}} |
||||
end) |
||||
|
||||
assert TokenInstanceMetadataRefetchOnDemand.trigger_refetch(token_instance) == :ok |
||||
|
||||
:timer.sleep(100) |
||||
|
||||
token_instance_from_db = |
||||
Repo.get_by(TokenInstance, token_id: token_id, token_contract_address_hash: token.contract_address_hash) |
||||
|
||||
assert(token_instance_from_db) |
||||
refute is_nil(token_instance_from_db.metadata) |
||||
|
||||
attempts = |
||||
Repo.get_by(TokenInstanceMetadataRefetchAttempt, |
||||
token_contract_address_hash: token.contract_address_hash, |
||||
token_id: token_id |
||||
) |
||||
|
||||
refute is_nil(attempts) |
||||
|
||||
assert attempts.retries_number == 1 |
||||
|
||||
refute_receive( |
||||
{:chain_event, :fetched_token_instance_metadata, :on_demand, |
||||
[^token_contract_address_hash_string, ^token_id, %{metadata: ^metadata}]} |
||||
) |
||||
|
||||
Application.put_env(:explorer, :http_adapter, HTTPoison) |
||||
end |
||||
end |
||||
end |
||||
Loading…
Reference in new issue