diff --git a/apps/explorer/test/explorer/smart_contract/reader_test.exs b/apps/explorer/test/explorer/smart_contract/reader_test.exs index 4b8a632c5d..184a28cbd8 100644 --- a/apps/explorer/test/explorer/smart_contract/reader_test.exs +++ b/apps/explorer/test/explorer/smart_contract/reader_test.exs @@ -131,6 +131,137 @@ defmodule Explorer.SmartContract.ReaderTest do assert response == %{"e72878b4" => {:ok, []}} end + + @abi [ + %{ + "type" => "function", + "stateMutability" => "view", + "payable" => false, + "outputs" => [ + %{"type" => "string", "name" => ""} + ], + "name" => "tokenURI", + "inputs" => [ + %{ + "type" => "uint256", + "name" => "_tokenId" + } + ], + "constant" => true + } + ] + + @abi_uri [ + %{ + "type" => "function", + "stateMutability" => "view", + "payable" => false, + "outputs" => [ + %{ + "type" => "string", + "name" => "", + "internalType" => "string" + } + ], + "name" => "uri", + "inputs" => [ + %{ + "type" => "uint256", + "name" => "_id", + "internalType" => "uint256" + } + ], + "constant" => true + } + ] + + test "fetches json metadata", %{json_rpc_named_arguments: json_rpc_named_arguments} do + if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do + EthereumJSONRPC.Mox + |> expect(:json_rpc, fn [ + %{ + id: 0, + jsonrpc: "2.0", + method: "eth_call", + params: [ + %{ + data: + "0xc87b56dd000000000000000000000000000000000000000000000000fdd5b9fa9d4bfb20", + to: "0x5caebd3b32e210e85ce3e9d51638b9c445481567" + }, + "latest" + ] + } + ], + _options -> + {:ok, + [ + %{ + id: 0, + jsonrpc: "2.0", + result: + "0x0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003568747470733a2f2f7661756c742e7761727269646572732e636f6d2f31383239303732393934373636373130323439362e6a736f6e0000000000000000000000" + } + ]} + end) + end + + assert %{ + "c87b56dd" => {:ok, ["https://vault.warriders.com/18290729947667102496.json"]} + } == + Reader.query_contract( + "0x5caebd3b32e210e85ce3e9d51638b9c445481567", + @abi, + %{ + "c87b56dd" => [18_290_729_947_667_102_496] + }, + false + ) + end + + test "fetches json metadata for ERC-1155 token", %{json_rpc_named_arguments: json_rpc_named_arguments} do + if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do + EthereumJSONRPC.Mox + |> expect(:json_rpc, fn [ + %{ + id: 0, + jsonrpc: "2.0", + method: "eth_call", + params: [ + %{ + data: + "0x0e89341c000000000000000000000000000000000000000000000000fdd5b9fa9d4bfb20", + to: "0x5caebd3b32e210e85ce3e9d51638b9c445481567" + }, + "latest" + ] + } + ], + _options -> + {:ok, + [ + %{ + id: 0, + jsonrpc: "2.0", + result: + "0x0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003568747470733a2f2f7661756c742e7761727269646572732e636f6d2f31383239303732393934373636373130323439362e6a736f6e0000000000000000000000" + } + ]} + end) + end + + assert %{ + "0e89341c" => {:ok, ["https://vault.warriders.com/18290729947667102496.json"]} + } == + Reader.query_contract( + "0x5caebd3b32e210e85ce3e9d51638b9c445481567", + @abi_uri, + %{ + "0e89341c" => [18_290_729_947_667_102_496] + }, + false + ) + end end describe "query_verified_contract/3" do diff --git a/apps/indexer/lib/indexer/fetcher/token_instance/metadata_retriever.ex b/apps/indexer/lib/indexer/fetcher/token_instance/metadata_retriever.ex index c1aa7e6da4..adb40d3389 100644 --- a/apps/indexer/lib/indexer/fetcher/token_instance/metadata_retriever.ex +++ b/apps/indexer/lib/indexer/fetcher/token_instance/metadata_retriever.ex @@ -6,12 +6,12 @@ defmodule Indexer.Fetcher.TokenInstance.MetadataRetriever do require Logger alias Explorer.Helper, as: ExplorerHelper - alias Explorer.SmartContract.Reader alias HTTPoison.{Error, Response} @no_uri_error "no uri" @vm_execution_error "VM execution error" @ipfs_protocol "ipfs://" + @invalid_base64_data "invalid data:application/json;base64" # https://eips.ethereum.org/EIPS/eip-1155#metadata @erc1155_token_id_placeholder "{id}" @@ -20,17 +20,48 @@ defmodule Indexer.Fetcher.TokenInstance.MetadataRetriever do @ignored_hosts ["localhost", "127.0.0.1", "0.0.0.0", "", nil] - defp ipfs_link do - link = + @spec ipfs_link(uid :: any()) :: String.t() + defp ipfs_link(uid) do + base_url = :indexer - |> Application.get_env(:ipfs_gateway_url) + |> Application.get_env(:ipfs) + |> Keyword.get(:gateway_url) |> String.trim_trailing("/") - link <> "/" + url = base_url <> "/" <> uid + + ipfs_params = Application.get_env(:indexer, :ipfs) + + if ipfs_params[:gateway_url_param_location] == :query do + gateway_url_param_key = ipfs_params[:gateway_url_param_key] + gateway_url_param_value = ipfs_params[:gateway_url_param_value] + + if gateway_url_param_key && gateway_url_param_value do + url <> "?#{gateway_url_param_key}=#{gateway_url_param_value}" + else + url + end + else + url + end end - def query_contract(contract_address_hash, contract_functions, abi) do - Reader.query_contract(contract_address_hash, abi, contract_functions, false) + @spec ipfs_headers() :: [{binary(), binary()}] + defp ipfs_headers do + ipfs_params = Application.get_env(:indexer, :ipfs) + + if ipfs_params[:gateway_url_param_location] == :header do + gateway_url_param_key = ipfs_params[:gateway_url_param_key] + gateway_url_param_value = ipfs_params[:gateway_url_param_value] + + if gateway_url_param_key && gateway_url_param_value do + [{gateway_url_param_key, gateway_url_param_value}] + else + [] + end + else + [] + end end @doc """ @@ -40,15 +71,17 @@ defmodule Indexer.Fetcher.TokenInstance.MetadataRetriever do {:error, binary} | {:error_code, any} | {:ok, %{metadata: any}} def fetch_json(uri, token_id \\ nil, hex_token_id \\ nil, from_base_uri? \\ false) - def fetch_json(uri, _token_id, _hex_token_id, _from_base_uri?) when uri in [{:ok, [""]}, {:ok, [""]}] do + def fetch_json({:ok, [""]}, _token_id, _hex_token_id, _from_base_uri?) do {:error, @no_uri_error} end def fetch_json(uri, token_id, hex_token_id, from_base_uri?) do - fetch_json_from_uri(uri, token_id, hex_token_id, from_base_uri?) + fetch_json_from_uri(uri, false, token_id, hex_token_id, from_base_uri?) end - defp fetch_json_from_uri({:error, error}, _token_id, _hex_token_id, _from_base_uri?) do + defp fetch_json_from_uri(_uri, _ipfs?, _token_id, _hex_token_id, _from_base_uri?) + + defp fetch_json_from_uri({:error, error}, _ipfs?, _token_id, _hex_token_id, _from_base_uri?) do error = to_string(error) if error =~ "execution reverted" or error =~ @vm_execution_error do @@ -62,9 +95,10 @@ defmodule Indexer.Fetcher.TokenInstance.MetadataRetriever do end # CIDv0 IPFS links # https://docs.ipfs.tech/concepts/content-addressing/#version-0-v0 - defp fetch_json_from_uri({:ok, ["Qm" <> _ = result]}, token_id, hex_token_id, from_base_uri?) do + defp fetch_json_from_uri({:ok, ["Qm" <> _ = result]}, _, token_id, hex_token_id, from_base_uri?) do if String.length(result) == 46 do - fetch_json_from_uri({:ok, [ipfs_link() <> result]}, token_id, hex_token_id, from_base_uri?) + ipfs? = true + fetch_json_from_uri({:ok, [ipfs_link(result)]}, ipfs?, token_id, hex_token_id, from_base_uri?) else Logger.warn(["Unknown metadata format result #{inspect(result)}."], fetcher: :token_instances) @@ -72,44 +106,52 @@ defmodule Indexer.Fetcher.TokenInstance.MetadataRetriever do end end - defp fetch_json_from_uri({:ok, ["'" <> token_uri]}, token_id, hex_token_id, from_base_uri?) do + defp fetch_json_from_uri({:ok, ["'" <> token_uri]}, ipfs?, token_id, hex_token_id, from_base_uri?) do token_uri = token_uri |> String.split("'") |> List.first() - fetch_metadata_inner(token_uri, token_id, hex_token_id, from_base_uri?) + fetch_metadata_inner(token_uri, ipfs?, token_id, hex_token_id, from_base_uri?) end - defp fetch_json_from_uri({:ok, ["http://" <> _ = token_uri]}, token_id, hex_token_id, from_base_uri?) do - fetch_metadata_inner(token_uri, token_id, hex_token_id, from_base_uri?) + defp fetch_json_from_uri({:ok, ["http://" <> _ = token_uri]}, ipfs?, token_id, hex_token_id, from_base_uri?) do + fetch_metadata_inner(token_uri, ipfs?, token_id, hex_token_id, from_base_uri?) end - defp fetch_json_from_uri({:ok, ["https://" <> _ = token_uri]}, token_id, hex_token_id, from_base_uri?) do - fetch_metadata_inner(token_uri, token_id, hex_token_id, from_base_uri?) + defp fetch_json_from_uri({:ok, ["https://" <> _ = token_uri]}, ipfs?, token_id, hex_token_id, from_base_uri?) do + fetch_metadata_inner(token_uri, ipfs?, token_id, hex_token_id, from_base_uri?) end defp fetch_json_from_uri( {:ok, [type = "data:application/json;utf8," <> json]}, + ipfs?, token_id, hex_token_id, from_base_uri? ) do - fetch_json_from_json_string(json, token_id, hex_token_id, from_base_uri?, type) + fetch_json_from_json_string(json, ipfs?, token_id, hex_token_id, from_base_uri?, type) end - defp fetch_json_from_uri({:ok, [type = "data:application/json," <> json]}, token_id, hex_token_id, from_base_uri?) do - fetch_json_from_json_string(json, token_id, hex_token_id, from_base_uri?, type) + defp fetch_json_from_uri( + {:ok, [type = "data:application/json," <> json]}, + ipfs?, + token_id, + hex_token_id, + from_base_uri? + ) do + fetch_json_from_json_string(json, ipfs?, token_id, hex_token_id, from_base_uri?, type) end defp fetch_json_from_uri( {:ok, ["data:application/json;base64," <> base64_encoded_json]}, + ipfs?, token_id, hex_token_id, from_base_uri? ) do case Base.decode64(base64_encoded_json) do {:ok, base64_decoded} -> - fetch_json_from_uri({:ok, [base64_decoded]}, token_id, hex_token_id, from_base_uri?) + fetch_json_from_uri({:ok, [base64_decoded]}, ipfs?, token_id, hex_token_id, from_base_uri?) _ -> - {:error, "invalid data:application/json;base64"} + {:error, @invalid_base64_data} end rescue e -> @@ -121,22 +163,22 @@ defmodule Indexer.Fetcher.TokenInstance.MetadataRetriever do fetcher: :token_instances ) - {:error, "invalid data:application/json;base64"} + {:error, @invalid_base64_data} end - defp fetch_json_from_uri({:ok, ["#{@ipfs_protocol}ipfs/" <> right]}, _token_id, hex_token_id, _from_base_uri?) do + defp fetch_json_from_uri({:ok, ["#{@ipfs_protocol}ipfs/" <> right]}, _ipfs?, _token_id, hex_token_id, _from_base_uri?) do fetch_from_ipfs(right, hex_token_id) end - defp fetch_json_from_uri({:ok, ["ipfs/" <> right]}, _token_id, hex_token_id, _from_base_uri?) do + defp fetch_json_from_uri({:ok, ["ipfs/" <> right]}, _ipfs?, _token_id, hex_token_id, _from_base_uri?) do fetch_from_ipfs(right, hex_token_id) end - defp fetch_json_from_uri({:ok, [@ipfs_protocol <> right]}, _token_id, hex_token_id, _from_base_uri?) do + defp fetch_json_from_uri({:ok, [@ipfs_protocol <> right]}, _ipfs?, _token_id, hex_token_id, _from_base_uri?) do fetch_from_ipfs(right, hex_token_id) end - defp fetch_json_from_uri({:ok, [json]}, _token_id, hex_token_id, _from_base_uri?) do + defp fetch_json_from_uri({:ok, [json]}, _ipfs?, _token_id, hex_token_id, _from_base_uri?) do json = ExplorerHelper.decode_json(json) check_type(json, hex_token_id) @@ -149,16 +191,16 @@ defmodule Indexer.Fetcher.TokenInstance.MetadataRetriever do {:error, "invalid json"} end - defp fetch_json_from_uri(uri, _token_id, _hex_token_id, _from_base_uri?) do + defp fetch_json_from_uri(uri, _ipfs?, _token_id, _hex_token_id, _from_base_uri?) do Logger.warn(["Unknown metadata uri format #{inspect(uri)}."], fetcher: :token_instances) {:error, "unknown metadata uri format"} end - defp fetch_json_from_json_string(json, token_id, hex_token_id, from_base_uri?, type) do + defp fetch_json_from_json_string(json, ipfs?, token_id, hex_token_id, from_base_uri?, type) do decoded_json = URI.decode(json) - fetch_json_from_uri({:ok, [decoded_json]}, token_id, hex_token_id, from_base_uri?) + fetch_json_from_uri({:ok, [decoded_json]}, ipfs?, token_id, hex_token_id, from_base_uri?) rescue e -> Logger.warn(["Unknown metadata format #{inspect(json)}.", Exception.format(:error, e, __STACKTRACE__)], @@ -169,15 +211,16 @@ defmodule Indexer.Fetcher.TokenInstance.MetadataRetriever do end defp fetch_from_ipfs(ipfs_uid, hex_token_id) do - ipfs_url = ipfs_link() <> ipfs_uid - fetch_metadata_inner(ipfs_url, nil, hex_token_id) + ipfs_url = ipfs_link(ipfs_uid) + ipfs? = true + fetch_metadata_inner(ipfs_url, ipfs?, nil, hex_token_id) end - defp fetch_metadata_inner(uri, token_id, hex_token_id, from_base_uri? \\ false) + defp fetch_metadata_inner(uri, ipfs?, token_id, hex_token_id, from_base_uri? \\ false) - defp fetch_metadata_inner(uri, token_id, hex_token_id, from_base_uri?) do + defp fetch_metadata_inner(uri, ipfs?, token_id, hex_token_id, from_base_uri?) do prepared_uri = substitute_token_id_to_token_uri(uri, token_id, hex_token_id, from_base_uri?) - fetch_metadata_from_uri(prepared_uri, hex_token_id) + fetch_metadata_from_uri(prepared_uri, ipfs?, hex_token_id) rescue e -> Logger.warn( @@ -188,24 +231,26 @@ defmodule Indexer.Fetcher.TokenInstance.MetadataRetriever do {:error, "preparation error"} end - def fetch_metadata_from_uri(uri, hex_token_id \\ nil) do + def fetch_metadata_from_uri(uri, ipfs?, hex_token_id \\ nil) do case Mix.env() != :test && URI.parse(uri) do %URI{host: host} when host in @ignored_hosts -> {:error, "ignored host #{host}"} _ -> - fetch_metadata_from_uri_inner(uri, hex_token_id) + fetch_metadata_from_uri_request(uri, hex_token_id, ipfs?) end end - def fetch_metadata_from_uri_inner(uri, hex_token_id) do - case Application.get_env(:explorer, :http_adapter).get(uri, [], + defp fetch_metadata_from_uri_request(uri, hex_token_id, ipfs?) do + headers = if ipfs?, do: ipfs_headers(), else: [] + + case Application.get_env(:explorer, :http_adapter).get(uri, headers, recv_timeout: 30_000, follow_redirect: true, hackney: [pool: :token_instance_fetcher] ) do - {:ok, %Response{body: body, status_code: 200, headers: headers}} -> - content_type = get_content_type_from_headers(headers) + {:ok, %Response{body: body, status_code: 200, headers: response_headers}} -> + content_type = get_content_type_from_headers(response_headers) check_content_type(content_type, uri, hex_token_id, body) @@ -311,5 +356,11 @@ defmodule Indexer.Fetcher.TokenInstance.MetadataRetriever do Truncate error string to @max_error_length symbols """ @spec truncate_error(binary()) :: binary() - def truncate_error(error), do: String.slice(error, 0, @max_error_length) + def truncate_error(error) do + if String.length(error) > @max_error_length - 2 do + String.slice(error, 0, @max_error_length - 3) <> "..." + else + error + end + end end diff --git a/apps/explorer/test/explorer/token/instance_metadata_retriever_test.exs b/apps/indexer/test/indexer/fetcher/token_instance/metadata_retriever_test.exs similarity index 65% rename from apps/explorer/test/explorer/token/instance_metadata_retriever_test.exs rename to apps/indexer/test/indexer/fetcher/token_instance/metadata_retriever_test.exs index d1d48cc654..071fa859ac 100644 --- a/apps/explorer/test/explorer/token/instance_metadata_retriever_test.exs +++ b/apps/indexer/test/indexer/fetcher/token_instance/metadata_retriever_test.exs @@ -1,4 +1,4 @@ -defmodule Explorer.Token.InstanceMetadataRetrieverTest do +defmodule Indexer.Fetcher.TokenInstance.MetadataRetrieverTest do use EthereumJSONRPC.Case alias Indexer.Fetcher.TokenInstance.MetadataRetriever @@ -9,144 +9,204 @@ defmodule Explorer.Token.InstanceMetadataRetrieverTest do setup :verify_on_exit! setup :set_mox_global - @abi [ - %{ - "type" => "function", - "stateMutability" => "view", - "payable" => false, - "outputs" => [ - %{"type" => "string", "name" => ""} - ], - "name" => "tokenURI", - "inputs" => [ - %{ - "type" => "uint256", - "name" => "_tokenId" - } - ], - "constant" => true - } - ] - - @abi_uri [ - %{ - "type" => "function", - "stateMutability" => "view", - "payable" => false, - "outputs" => [ - %{ - "type" => "string", - "name" => "", - "internalType" => "string" - } - ], - "name" => "uri", - "inputs" => [ - %{ - "type" => "uint256", - "name" => "_id", - "internalType" => "uint256" - } - ], - "constant" => true - } - ] - - describe "fetch_metadata/2" do - @tag :no_nethermind - @tag :no_geth - test "fetches json metadata", %{json_rpc_named_arguments: json_rpc_named_arguments} do - if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do - EthereumJSONRPC.Mox - |> expect(:json_rpc, fn [ - %{ - id: 0, - jsonrpc: "2.0", - method: "eth_call", - params: [ - %{ - data: - "0xc87b56dd000000000000000000000000000000000000000000000000fdd5b9fa9d4bfb20", - to: "0x5caebd3b32e210e85ce3e9d51638b9c445481567" - }, - "latest" - ] - } - ], - _options -> - {:ok, - [ - %{ - id: 0, - jsonrpc: "2.0", - result: - "0x0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003568747470733a2f2f7661756c742e7761727269646572732e636f6d2f31383239303732393934373636373130323439362e6a736f6e0000000000000000000000" - } - ]} - end) - end - - assert %{ - "c87b56dd" => {:ok, ["https://vault.warriders.com/18290729947667102496.json"]} - } == - MetadataRetriever.query_contract( - "0x5caebd3b32e210e85ce3e9d51638b9c445481567", - %{ - "c87b56dd" => [18_290_729_947_667_102_496] - }, - @abi - ) + describe "fetch_json/4" do + setup do + bypass = Bypass.open() + + {:ok, bypass: bypass} end - test "fetches json metadata for ERC-1155 token", %{json_rpc_named_arguments: json_rpc_named_arguments} do - if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do - EthereumJSONRPC.Mox - |> expect(:json_rpc, fn [ - %{ - id: 0, - jsonrpc: "2.0", - method: "eth_call", - params: [ - %{ - data: - "0x0e89341c000000000000000000000000000000000000000000000000fdd5b9fa9d4bfb20", - to: "0x5caebd3b32e210e85ce3e9d51638b9c445481567" - }, - "latest" - ] - } - ], - _options -> - {:ok, - [ - %{ - id: 0, - jsonrpc: "2.0", - result: - "0x0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003568747470733a2f2f7661756c742e7761727269646572732e636f6d2f31383239303732393934373636373130323439362e6a736f6e0000000000000000000000" - } - ]} - end) - end - - assert %{ - "0e89341c" => {:ok, ["https://vault.warriders.com/18290729947667102496.json"]} - } == - MetadataRetriever.query_contract( - "0x5caebd3b32e210e85ce3e9d51638b9c445481567", - %{ - "0e89341c" => [18_290_729_947_667_102_496] - }, - @abi_uri - ) + test "returns {:error, @no_uri_error} when empty uri is passed" do + error = {:error, "no uri"} + token_id = "TOKEN_ID" + hex_token_id = "HEX_TOKEN_ID" + from_base_uri = true + + result = MetadataRetriever.fetch_json({:ok, [""]}, token_id, hex_token_id, from_base_uri) + + assert result == error end - end - describe "fetch_json/1" do - setup do - bypass = Bypass.open() + test "returns {:error, @vm_execution_error} when 'execution reverted' error passed in uri" do + uri_error = {:error, "something happened: execution reverted"} + token_id = "TOKEN_ID" + hex_token_id = "HEX_TOKEN_ID" + from_base_uri = true + result_error = {:error, "VM execution error"} - {:ok, bypass: bypass} + result = MetadataRetriever.fetch_json(uri_error, token_id, hex_token_id, from_base_uri) + + assert result == result_error + end + + test "returns {:error, @vm_execution_error} when 'VM execution error' error passed in uri" do + error = {:error, "VM execution error"} + token_id = "TOKEN_ID" + hex_token_id = "HEX_TOKEN_ID" + from_base_uri = true + + result = MetadataRetriever.fetch_json(error, token_id, hex_token_id, from_base_uri) + + assert result == error + end + + test "returns {:error, error} when all other errors passed in uri" do + error = {:error, "Some error"} + token_id = "TOKEN_ID" + hex_token_id = "HEX_TOKEN_ID" + from_base_uri = true + + result = MetadataRetriever.fetch_json(error, token_id, hex_token_id, from_base_uri) + + assert result == error + end + + test "returns {:error, truncated_error} when long error passed in uri" do + error = + {:error, + "ERROR: Unable to establish a connection to the database server. The database server may be offline, or there could be a network issue preventing access. Please ensure that the database server is running and that the network configuration is correct. Additionally, check the database credentials and permissions to ensure they are valid. If the issue persists, contact your system administrator for further assistance. Error code: DB_CONN_FAILED_101234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"} + + token_id = "TOKEN_ID" + hex_token_id = "HEX_TOKEN_ID" + from_base_uri = true + + truncated_error = + {:error, + "ERROR: Unable to establish a connection to the database server. The database server may be offline, or there could be a network issue preventing access. Please ensure that the database server is running and that the network configuration is correct. Ad..."} + + result = MetadataRetriever.fetch_json(error, token_id, hex_token_id, from_base_uri) + + assert result == truncated_error + end + + test "Constructs IPFS link with query param" do + configuration = Application.get_env(:indexer, :ipfs) + + Application.put_env(:indexer, :ipfs, + gateway_url: Keyword.get(configuration, :gateway_url), + gateway_url_param_location: :query, + gateway_url_param_key: "x-apikey", + gateway_url_param_value: "mykey" + ) + + data = "QmT1Yz43R1PLn2RVovAnEM5dHQEvpTcnwgX8zftvY1FcjP" + + result = %{ + "name" => "asda", + "description" => "asda", + "salePrice" => 34, + "img_hash" => "QmUfW3PVnh9GGuHcQgc3ZeNEbhwp5HE8rS5ac9MDWWQebz", + "collectionId" => "1871_1665123820823" + } + + Application.put_env(:explorer, :http_adapter, Explorer.Mox.HTTPoison) + + Explorer.Mox.HTTPoison + |> expect(:get, fn "https://ipfs.io/ipfs/QmT1Yz43R1PLn2RVovAnEM5dHQEvpTcnwgX8zftvY1FcjP?x-apikey=mykey", + _headers, + _options -> + {:ok, %HTTPoison.Response{status_code: 200, body: Jason.encode!(result)}} + end) + + assert {:ok, + %{ + metadata: %{ + "collectionId" => "1871_1665123820823", + "description" => "asda", + "img_hash" => "QmUfW3PVnh9GGuHcQgc3ZeNEbhwp5HE8rS5ac9MDWWQebz", + "name" => "asda", + "salePrice" => 34 + } + }} == MetadataRetriever.fetch_json({:ok, [data]}) + + Application.put_env(:explorer, :http_adapter, HTTPoison) + Application.put_env(:indexer, :ipfs, configuration) + end + + test "Constructs IPFS link with no query param, if gateway_url_param_location is invalid" do + configuration = Application.get_env(:indexer, :ipfs) + + Application.put_env(:indexer, :ipfs, + gateway_url: Keyword.get(configuration, :gateway_url), + gateway_url_param_location: :query2, + gateway_url_param_key: "x-apikey", + gateway_url_param_value: "mykey" + ) + + data = "QmT1Yz43R1PLn2RVovAnEM5dHQEvpTcnwgX8zftvY1FcjP" + + result = %{ + "name" => "asda", + "description" => "asda", + "salePrice" => 34, + "img_hash" => "QmUfW3PVnh9GGuHcQgc3ZeNEbhwp5HE8rS5ac9MDWWQebz", + "collectionId" => "1871_1665123820823" + } + + Application.put_env(:explorer, :http_adapter, Explorer.Mox.HTTPoison) + + Explorer.Mox.HTTPoison + |> expect(:get, fn "https://ipfs.io/ipfs/QmT1Yz43R1PLn2RVovAnEM5dHQEvpTcnwgX8zftvY1FcjP", _headers, _options -> + {:ok, %HTTPoison.Response{status_code: 200, body: Jason.encode!(result)}} + end) + + assert {:ok, + %{ + metadata: %{ + "collectionId" => "1871_1665123820823", + "description" => "asda", + "img_hash" => "QmUfW3PVnh9GGuHcQgc3ZeNEbhwp5HE8rS5ac9MDWWQebz", + "name" => "asda", + "salePrice" => 34 + } + }} == MetadataRetriever.fetch_json({:ok, [data]}) + + Application.put_env(:explorer, :http_adapter, HTTPoison) + Application.put_env(:indexer, :ipfs, configuration) + end + + test "Constructs IPFS link with additional header" do + configuration = Application.get_env(:indexer, :ipfs) + + Application.put_env(:indexer, :ipfs, + gateway_url: Keyword.get(configuration, :gateway_url), + gateway_url_param_location: :header, + gateway_url_param_key: "x-apikey", + gateway_url_param_value: "mykey" + ) + + data = "QmT1Yz43R1PLn2RVovAnEM5dHQEvpTcnwgX8zftvY1FcjP" + + result = %{ + "name" => "asda", + "description" => "asda", + "salePrice" => 34, + "img_hash" => "QmUfW3PVnh9GGuHcQgc3ZeNEbhwp5HE8rS5ac9MDWWQebz", + "collectionId" => "1871_1665123820823" + } + + Application.put_env(:explorer, :http_adapter, Explorer.Mox.HTTPoison) + + Explorer.Mox.HTTPoison + |> expect(:get, fn "https://ipfs.io/ipfs/QmT1Yz43R1PLn2RVovAnEM5dHQEvpTcnwgX8zftvY1FcjP", + [{"x-apikey", "mykey"}], + _options -> + {:ok, %HTTPoison.Response{status_code: 200, body: Jason.encode!(result)}} + end) + + assert {:ok, + %{ + metadata: %{ + "collectionId" => "1871_1665123820823", + "description" => "asda", + "img_hash" => "QmUfW3PVnh9GGuHcQgc3ZeNEbhwp5HE8rS5ac9MDWWQebz", + "name" => "asda", + "salePrice" => 34 + } + }} == MetadataRetriever.fetch_json({:ok, [data]}) + + Application.put_env(:explorer, :http_adapter, HTTPoison) + Application.put_env(:indexer, :ipfs, configuration) end test "fetches json with latin1 encoding", %{bypass: bypass} do @@ -190,7 +250,8 @@ defmodule Explorer.Token.InstanceMetadataRetrieverTest do Conn.resp(conn, 200, json) end) - {:ok, %{metadata: metadata}} = MetadataRetriever.fetch_metadata_from_uri("http://localhost:#{bypass.port}#{path}") + {:ok, %{metadata: metadata}} = + MetadataRetriever.fetch_metadata_from_uri("http://localhost:#{bypass.port}#{path}", []) assert Map.get(metadata, "attributes") == Jason.decode!(attributes) end diff --git a/config/config_helper.exs b/config/config_helper.exs index a703ac522b..b00f9c0b2b 100644 --- a/config/config_helper.exs +++ b/config/config_helper.exs @@ -1,4 +1,6 @@ defmodule ConfigHelper do + require Logger + import Bitwise alias Explorer.ExchangeRates.Source alias Explorer.Market.History.Source.{MarketCap, Price, TVL} @@ -95,6 +97,35 @@ defmodule ConfigHelper do end end + @doc """ + Parses value of env var through catalogued values list. If a value is not in the list, nil is returned. + Also, the application shutdown option is supported, if a value is wrong. + """ + @spec parse_catalog_value(String.t(), List.t(), bool(), String.t() | nil) :: atom() | nil + def parse_catalog_value(env_var, catalog, shutdown_on_wrong_value?, default_value \\ nil) do + value = env_var |> safe_get_env(default_value) + + if value !== "" do + if value in catalog do + String.to_atom(value) + else + if shutdown_on_wrong_value? do + Logger.error(wrong_value_error(value, env_var, catalog)) + exit(:shutdown) + else + Logger.warning(wrong_value_error(value, env_var, catalog)) + nil + end + end + else + nil + end + end + + defp wrong_value_error(value, env_var, catalog) do + "Wrong value #{value} of #{env_var} environment variable. Supported values are #{inspect(catalog)}" + end + def safe_get_env(env_var, default_value) do env_var |> System.get_env(default_value) diff --git a/config/runtime.exs b/config/runtime.exs index 016f10c36c..99ddd5d614 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -598,9 +598,15 @@ config :indexer, ConfigHelper.parse_integer_env_var("INDEXER_TOKEN_BALANCES_FETCHER_INIT_QUERY_LIMIT", 100_000), coin_balances_fetcher_init_limit: ConfigHelper.parse_integer_env_var("INDEXER_COIN_BALANCES_FETCHER_INIT_QUERY_LIMIT", 2000), - ipfs_gateway_url: System.get_env("IPFS_GATEWAY_URL", "https://ipfs.io/ipfs"), graceful_shutdown_period: ConfigHelper.parse_time_env_var("INDEXER_GRACEFUL_SHUTDOWN_PERIOD", "5m") +config :indexer, :ipfs, + gateway_url: System.get_env("IPFS_GATEWAY_URL", "https://ipfs.io/ipfs"), + gateway_url_param_key: System.get_env("IPFS_GATEWAY_URL_PARAM_KEY"), + gateway_url_param_value: System.get_env("IPFS_GATEWAY_URL_PARAM_VALUE"), + gateway_url_param_location: + ConfigHelper.parse_catalog_value("IPFS_GATEWAY_URL_PARAM_LOCATION", ["query", "header"], true) + config :indexer, Indexer.Supervisor, enabled: !ConfigHelper.parse_bool_env_var("DISABLE_INDEXER") config :indexer, Indexer.Fetcher.TransactionAction.Supervisor, diff --git a/cspell.json b/cspell.json index 8c7c0c36ab..c22d84c555 100644 --- a/cspell.json +++ b/cspell.json @@ -368,6 +368,7 @@ "munknownc", "munknowne", "mydep", + "mykey", "nanomorph", "nbsp", "newkey",