IPFS gateway URL extra params (#9898)

* IPFS gateway URL authentication params

* Fix cspell

* Fix explorer tests

* Add IPFS gateway URL custom header support

* Parse cataloged value of IPFS_GATEWAY_URL_PARAM_LOCATION

* Update apps/indexer/lib/indexer/fetcher/token_instance/metadata_retriever.ex

Co-authored-by: Qwerty5Uiop <105209995+Qwerty5Uiop@users.noreply.github.com>

* Update apps/indexer/lib/indexer/fetcher/token_instance/metadata_retriever.ex

Co-authored-by: Qwerty5Uiop <105209995+Qwerty5Uiop@users.noreply.github.com>

* Process review comments

* Return exit(:shutdown)

---------

Co-authored-by: Qwerty5Uiop <105209995+Qwerty5Uiop@users.noreply.github.com>

apps/explorer/test/explorer/smart_contract/reader_test.exs

@@ -131,6 +131,137 @@ defmodule Explorer.SmartContract.ReaderTest do
 
       assert response == %{"e72878b4" => {:ok, []}}
     end
+
+    @abi [
+      %{
+        "type" => "function",
+        "stateMutability" => "view",
+        "payable" => false,
+        "outputs" => [
+          %{"type" => "string", "name" => ""}
+        ],
+        "name" => "tokenURI",
+        "inputs" => [
+          %{
+            "type" => "uint256",
+            "name" => "_tokenId"
+          }
+        ],
+        "constant" => true
+      }
+    ]
+
+    @abi_uri [
+      %{
+        "type" => "function",
+        "stateMutability" => "view",
+        "payable" => false,
+        "outputs" => [
+          %{
+            "type" => "string",
+            "name" => "",
+            "internalType" => "string"
+          }
+        ],
+        "name" => "uri",
+        "inputs" => [
+          %{
+            "type" => "uint256",
+            "name" => "_id",
+            "internalType" => "uint256"
+          }
+        ],
+        "constant" => true
+      }
+    ]
+
+    test "fetches json metadata", %{json_rpc_named_arguments: json_rpc_named_arguments} do
+      if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do
+        EthereumJSONRPC.Mox
+        |> expect(:json_rpc, fn [
+                                  %{
+                                    id: 0,
+                                    jsonrpc: "2.0",
+                                    method: "eth_call",
+                                    params: [
+                                      %{
+                                        data:
+                                          "0xc87b56dd000000000000000000000000000000000000000000000000fdd5b9fa9d4bfb20",
+                                        to: "0x5caebd3b32e210e85ce3e9d51638b9c445481567"
+                                      },
+                                      "latest"
+                                    ]
+                                  }
+                                ],
+                                _options ->
+          {:ok,
+           [
+             %{
+               id: 0,
+               jsonrpc: "2.0",
+               result:
+                 "0x0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003568747470733a2f2f7661756c742e7761727269646572732e636f6d2f31383239303732393934373636373130323439362e6a736f6e0000000000000000000000"
+             }
+           ]}
+        end)
+      end
+
+      assert %{
+               "c87b56dd" => {:ok, ["https://vault.warriders.com/18290729947667102496.json"]}
+             } ==
+               Reader.query_contract(
+                 "0x5caebd3b32e210e85ce3e9d51638b9c445481567",
+                 @abi,
+                 %{
+                   "c87b56dd" => [18_290_729_947_667_102_496]
+                 },
+                 false
+               )
+    end
+
+    test "fetches json metadata for ERC-1155 token", %{json_rpc_named_arguments: json_rpc_named_arguments} do
+      if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do
+        EthereumJSONRPC.Mox
+        |> expect(:json_rpc, fn [
+                                  %{
+                                    id: 0,
+                                    jsonrpc: "2.0",
+                                    method: "eth_call",
+                                    params: [
+                                      %{
+                                        data:
+                                          "0x0e89341c000000000000000000000000000000000000000000000000fdd5b9fa9d4bfb20",
+                                        to: "0x5caebd3b32e210e85ce3e9d51638b9c445481567"
+                                      },
+                                      "latest"
+                                    ]
+                                  }
+                                ],
+                                _options ->
+          {:ok,
+           [
+             %{
+               id: 0,
+               jsonrpc: "2.0",
+               result:
+                 "0x0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003568747470733a2f2f7661756c742e7761727269646572732e636f6d2f31383239303732393934373636373130323439362e6a736f6e0000000000000000000000"
+             }
+           ]}
+        end)
+      end
+
+      assert %{
+               "0e89341c" => {:ok, ["https://vault.warriders.com/18290729947667102496.json"]}
+             } ==
+               Reader.query_contract(
+                 "0x5caebd3b32e210e85ce3e9d51638b9c445481567",
+                 @abi_uri,
+                 %{
+                   "0e89341c" => [18_290_729_947_667_102_496]
+                 },
+                 false
+               )
+    end
   end
 
   describe "query_verified_contract/3" do

apps/indexer/lib/indexer/fetcher/token_instance/metadata_retriever.ex

@@ -6,12 +6,12 @@ defmodule Indexer.Fetcher.TokenInstance.MetadataRetriever do
   require Logger
 
   alias Explorer.Helper, as: ExplorerHelper
-  alias Explorer.SmartContract.Reader
   alias HTTPoison.{Error, Response}
 
   @no_uri_error "no uri"
   @vm_execution_error "VM execution error"
   @ipfs_protocol "ipfs://"
+  @invalid_base64_data "invalid data:application/json;base64"
 
   # https://eips.ethereum.org/EIPS/eip-1155#metadata
   @erc1155_token_id_placeholder "{id}"
@@ -20,17 +20,48 @@ defmodule Indexer.Fetcher.TokenInstance.MetadataRetriever do
 
   @ignored_hosts ["localhost", "127.0.0.1", "0.0.0.0", "", nil]
 
-  defp ipfs_link do
-    link =
+  @spec ipfs_link(uid :: any()) :: String.t()
+  defp ipfs_link(uid) do
+    base_url =
       :indexer
-      |> Application.get_env(:ipfs_gateway_url)
+      |> Application.get_env(:ipfs)
+      |> Keyword.get(:gateway_url)
       |> String.trim_trailing("/")
 
-    link <> "/"
+    url = base_url <> "/" <> uid
+
+    ipfs_params = Application.get_env(:indexer, :ipfs)
+
+    if ipfs_params[:gateway_url_param_location] == :query do
+      gateway_url_param_key = ipfs_params[:gateway_url_param_key]
+      gateway_url_param_value = ipfs_params[:gateway_url_param_value]
+
+      if gateway_url_param_key && gateway_url_param_value do
+        url <> "?#{gateway_url_param_key}=#{gateway_url_param_value}"
+      else
+        url
+      end
+    else
+      url
     end
+  end
+
+  @spec ipfs_headers() :: [{binary(), binary()}]
+  defp ipfs_headers do
+    ipfs_params = Application.get_env(:indexer, :ipfs)
+
+    if ipfs_params[:gateway_url_param_location] == :header do
+      gateway_url_param_key = ipfs_params[:gateway_url_param_key]
+      gateway_url_param_value = ipfs_params[:gateway_url_param_value]
 
-  def query_contract(contract_address_hash, contract_functions, abi) do
-    Reader.query_contract(contract_address_hash, abi, contract_functions, false)
+      if gateway_url_param_key && gateway_url_param_value do
+        [{gateway_url_param_key, gateway_url_param_value}]
+      else
+        []
+      end
+    else
+      []
+    end
   end
 
   @doc """
@@ -40,15 +71,17 @@ defmodule Indexer.Fetcher.TokenInstance.MetadataRetriever do
           {:error, binary} | {:error_code, any} | {:ok, %{metadata: any}}
   def fetch_json(uri, token_id \\ nil, hex_token_id \\ nil, from_base_uri? \\ false)
 
-  def fetch_json(uri, _token_id, _hex_token_id, _from_base_uri?) when uri in [{:ok, [""]}, {:ok, [""]}] do
+  def fetch_json({:ok, [""]}, _token_id, _hex_token_id, _from_base_uri?) do
     {:error, @no_uri_error}
   end
 
   def fetch_json(uri, token_id, hex_token_id, from_base_uri?) do
-    fetch_json_from_uri(uri, token_id, hex_token_id, from_base_uri?)
+    fetch_json_from_uri(uri, false, token_id, hex_token_id, from_base_uri?)
   end
 
-  defp fetch_json_from_uri({:error, error}, _token_id, _hex_token_id, _from_base_uri?) do
+  defp fetch_json_from_uri(_uri, _ipfs?, _token_id, _hex_token_id, _from_base_uri?)
+
+  defp fetch_json_from_uri({:error, error}, _ipfs?, _token_id, _hex_token_id, _from_base_uri?) do
     error = to_string(error)
 
     if error =~ "execution reverted" or error =~ @vm_execution_error do
@@ -62,9 +95,10 @@ defmodule Indexer.Fetcher.TokenInstance.MetadataRetriever do
   end
 
   # CIDv0 IPFS links # https://docs.ipfs.tech/concepts/content-addressing/#version-0-v0
-  defp fetch_json_from_uri({:ok, ["Qm" <> _ = result]}, token_id, hex_token_id, from_base_uri?) do
+  defp fetch_json_from_uri({:ok, ["Qm" <> _ = result]}, _, token_id, hex_token_id, from_base_uri?) do
     if String.length(result) == 46 do
-      fetch_json_from_uri({:ok, [ipfs_link() <> result]}, token_id, hex_token_id, from_base_uri?)
+      ipfs? = true
+      fetch_json_from_uri({:ok, [ipfs_link(result)]}, ipfs?, token_id, hex_token_id, from_base_uri?)
     else
       Logger.warn(["Unknown metadata format result #{inspect(result)}."], fetcher: :token_instances)
 
@@ -72,44 +106,52 @@ defmodule Indexer.Fetcher.TokenInstance.MetadataRetriever do
     end
   end
 
-  defp fetch_json_from_uri({:ok, ["'" <> token_uri]}, token_id, hex_token_id, from_base_uri?) do
+  defp fetch_json_from_uri({:ok, ["'" <> token_uri]}, ipfs?, token_id, hex_token_id, from_base_uri?) do
     token_uri = token_uri |> String.split("'") |> List.first()
-    fetch_metadata_inner(token_uri, token_id, hex_token_id, from_base_uri?)
+    fetch_metadata_inner(token_uri, ipfs?, token_id, hex_token_id, from_base_uri?)
   end
 
-  defp fetch_json_from_uri({:ok, ["http://" <> _ = token_uri]}, token_id, hex_token_id, from_base_uri?) do
-    fetch_metadata_inner(token_uri, token_id, hex_token_id, from_base_uri?)
+  defp fetch_json_from_uri({:ok, ["http://" <> _ = token_uri]}, ipfs?, token_id, hex_token_id, from_base_uri?) do
+    fetch_metadata_inner(token_uri, ipfs?, token_id, hex_token_id, from_base_uri?)
   end
 
-  defp fetch_json_from_uri({:ok, ["https://" <> _ = token_uri]}, token_id, hex_token_id, from_base_uri?) do
-    fetch_metadata_inner(token_uri, token_id, hex_token_id, from_base_uri?)
+  defp fetch_json_from_uri({:ok, ["https://" <> _ = token_uri]}, ipfs?, token_id, hex_token_id, from_base_uri?) do
+    fetch_metadata_inner(token_uri, ipfs?, token_id, hex_token_id, from_base_uri?)
   end
 
   defp fetch_json_from_uri(
          {:ok, [type = "data:application/json;utf8," <> json]},
+         ipfs?,
          token_id,
          hex_token_id,
          from_base_uri?
        ) do
-    fetch_json_from_json_string(json, token_id, hex_token_id, from_base_uri?, type)
+    fetch_json_from_json_string(json, ipfs?, token_id, hex_token_id, from_base_uri?, type)
   end
 
-  defp fetch_json_from_uri({:ok, [type = "data:application/json," <> json]}, token_id, hex_token_id, from_base_uri?) do
-    fetch_json_from_json_string(json, token_id, hex_token_id, from_base_uri?, type)
+  defp fetch_json_from_uri(
+         {:ok, [type = "data:application/json," <> json]},
+         ipfs?,
+         token_id,
+         hex_token_id,
+         from_base_uri?
+       ) do
+    fetch_json_from_json_string(json, ipfs?, token_id, hex_token_id, from_base_uri?, type)
   end
 
   defp fetch_json_from_uri(
          {:ok, ["data:application/json;base64," <> base64_encoded_json]},
+         ipfs?,
          token_id,
          hex_token_id,
          from_base_uri?
        ) do
     case Base.decode64(base64_encoded_json) do
       {:ok, base64_decoded} ->
-        fetch_json_from_uri({:ok, [base64_decoded]}, token_id, hex_token_id, from_base_uri?)
+        fetch_json_from_uri({:ok, [base64_decoded]}, ipfs?, token_id, hex_token_id, from_base_uri?)
 
       _ ->
-        {:error, "invalid data:application/json;base64"}
+        {:error, @invalid_base64_data}
     end
   rescue
     e ->
@@ -121,22 +163,22 @@ defmodule Indexer.Fetcher.TokenInstance.MetadataRetriever do
         fetcher: :token_instances
       )
 
-      {:error, "invalid data:application/json;base64"}
+      {:error, @invalid_base64_data}
   end
 
-  defp fetch_json_from_uri({:ok, ["#{@ipfs_protocol}ipfs/" <> right]}, _token_id, hex_token_id, _from_base_uri?) do
+  defp fetch_json_from_uri({:ok, ["#{@ipfs_protocol}ipfs/" <> right]}, _ipfs?, _token_id, hex_token_id, _from_base_uri?) do
     fetch_from_ipfs(right, hex_token_id)
   end
 
-  defp fetch_json_from_uri({:ok, ["ipfs/" <> right]}, _token_id, hex_token_id, _from_base_uri?) do
+  defp fetch_json_from_uri({:ok, ["ipfs/" <> right]}, _ipfs?, _token_id, hex_token_id, _from_base_uri?) do
     fetch_from_ipfs(right, hex_token_id)
   end
 
-  defp fetch_json_from_uri({:ok, [@ipfs_protocol <> right]}, _token_id, hex_token_id, _from_base_uri?) do
+  defp fetch_json_from_uri({:ok, [@ipfs_protocol <> right]}, _ipfs?, _token_id, hex_token_id, _from_base_uri?) do
     fetch_from_ipfs(right, hex_token_id)
   end
 
-  defp fetch_json_from_uri({:ok, [json]}, _token_id, hex_token_id, _from_base_uri?) do
+  defp fetch_json_from_uri({:ok, [json]}, _ipfs?, _token_id, hex_token_id, _from_base_uri?) do
     json = ExplorerHelper.decode_json(json)
 
     check_type(json, hex_token_id)
@@ -149,16 +191,16 @@ defmodule Indexer.Fetcher.TokenInstance.MetadataRetriever do
       {:error, "invalid json"}
   end
 
-  defp fetch_json_from_uri(uri, _token_id, _hex_token_id, _from_base_uri?) do
+  defp fetch_json_from_uri(uri, _ipfs?, _token_id, _hex_token_id, _from_base_uri?) do
     Logger.warn(["Unknown metadata uri format #{inspect(uri)}."], fetcher: :token_instances)
 
     {:error, "unknown metadata uri format"}
   end
 
-  defp fetch_json_from_json_string(json, token_id, hex_token_id, from_base_uri?, type) do
+  defp fetch_json_from_json_string(json, ipfs?, token_id, hex_token_id, from_base_uri?, type) do
     decoded_json = URI.decode(json)
 
-    fetch_json_from_uri({:ok, [decoded_json]}, token_id, hex_token_id, from_base_uri?)
+    fetch_json_from_uri({:ok, [decoded_json]}, ipfs?, token_id, hex_token_id, from_base_uri?)
   rescue
     e ->
       Logger.warn(["Unknown metadata format #{inspect(json)}.", Exception.format(:error, e, __STACKTRACE__)],
@@ -169,15 +211,16 @@ defmodule Indexer.Fetcher.TokenInstance.MetadataRetriever do
   end
 
   defp fetch_from_ipfs(ipfs_uid, hex_token_id) do
-    ipfs_url = ipfs_link() <> ipfs_uid
-    fetch_metadata_inner(ipfs_url, nil, hex_token_id)
+    ipfs_url = ipfs_link(ipfs_uid)
+    ipfs? = true
+    fetch_metadata_inner(ipfs_url, ipfs?, nil, hex_token_id)
   end
 
-  defp fetch_metadata_inner(uri, token_id, hex_token_id, from_base_uri? \\ false)
+  defp fetch_metadata_inner(uri, ipfs?, token_id, hex_token_id, from_base_uri? \\ false)
 
-  defp fetch_metadata_inner(uri, token_id, hex_token_id, from_base_uri?) do
+  defp fetch_metadata_inner(uri, ipfs?, token_id, hex_token_id, from_base_uri?) do
     prepared_uri = substitute_token_id_to_token_uri(uri, token_id, hex_token_id, from_base_uri?)
-    fetch_metadata_from_uri(prepared_uri, hex_token_id)
+    fetch_metadata_from_uri(prepared_uri, ipfs?, hex_token_id)
   rescue
     e ->
       Logger.warn(
@@ -188,24 +231,26 @@ defmodule Indexer.Fetcher.TokenInstance.MetadataRetriever do
       {:error, "preparation error"}
   end
 
-  def fetch_metadata_from_uri(uri, hex_token_id \\ nil) do
+  def fetch_metadata_from_uri(uri, ipfs?, hex_token_id \\ nil) do
     case Mix.env() != :test && URI.parse(uri) do
       %URI{host: host} when host in @ignored_hosts ->
         {:error, "ignored host #{host}"}
 
       _ ->
-        fetch_metadata_from_uri_inner(uri, hex_token_id)
+        fetch_metadata_from_uri_request(uri, hex_token_id, ipfs?)
     end
   end
 
-  def fetch_metadata_from_uri_inner(uri, hex_token_id) do
-    case Application.get_env(:explorer, :http_adapter).get(uri, [],
+  defp fetch_metadata_from_uri_request(uri, hex_token_id, ipfs?) do
+    headers = if ipfs?, do: ipfs_headers(), else: []
+
+    case Application.get_env(:explorer, :http_adapter).get(uri, headers,
            recv_timeout: 30_000,
            follow_redirect: true,
            hackney: [pool: :token_instance_fetcher]
          ) do
-      {:ok, %Response{body: body, status_code: 200, headers: headers}} ->
-        content_type = get_content_type_from_headers(headers)
+      {:ok, %Response{body: body, status_code: 200, headers: response_headers}} ->
+        content_type = get_content_type_from_headers(response_headers)
 
         check_content_type(content_type, uri, hex_token_id, body)
 
@@ -311,5 +356,11 @@ defmodule Indexer.Fetcher.TokenInstance.MetadataRetriever do
     Truncate error string to @max_error_length symbols
   """
   @spec truncate_error(binary()) :: binary()
-  def truncate_error(error), do: String.slice(error, 0, @max_error_length)
+  def truncate_error(error) do
+    if String.length(error) > @max_error_length - 2 do
+      String.slice(error, 0, @max_error_length - 3) <> "..."
+    else
+      error
+    end
+  end
 end

apps/indexer/test/indexer/fetcher/token_instance/metadata_retriever_test.exs

@@ -1,4 +1,4 @@
-defmodule Explorer.Token.InstanceMetadataRetrieverTest do
+defmodule Indexer.Fetcher.TokenInstance.MetadataRetrieverTest do
   use EthereumJSONRPC.Case
 
   alias Indexer.Fetcher.TokenInstance.MetadataRetriever
@@ -9,144 +9,204 @@ defmodule Explorer.Token.InstanceMetadataRetrieverTest do
   setup :verify_on_exit!
   setup :set_mox_global
 
-  @abi [
-    %{
-      "type" => "function",
-      "stateMutability" => "view",
-      "payable" => false,
-      "outputs" => [
-        %{"type" => "string", "name" => ""}
-      ],
-      "name" => "tokenURI",
-      "inputs" => [
-        %{
-          "type" => "uint256",
-          "name" => "_tokenId"
-        }
-      ],
-      "constant" => true
-    }
-  ]
+  describe "fetch_json/4" do
+    setup do
+      bypass = Bypass.open()
 
-  @abi_uri [
-    %{
-      "type" => "function",
-      "stateMutability" => "view",
-      "payable" => false,
-      "outputs" => [
-        %{
-          "type" => "string",
-          "name" => "",
-          "internalType" => "string"
-        }
-      ],
-      "name" => "uri",
-      "inputs" => [
-        %{
-          "type" => "uint256",
-          "name" => "_id",
-          "internalType" => "uint256"
-        }
-      ],
-      "constant" => true
-    }
-  ]
+      {:ok, bypass: bypass}
+    end
 
-  describe "fetch_metadata/2" do
-    @tag :no_nethermind
-    @tag :no_geth
-    test "fetches json metadata", %{json_rpc_named_arguments: json_rpc_named_arguments} do
-      if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do
-        EthereumJSONRPC.Mox
-        |> expect(:json_rpc, fn [
-                                  %{
-                                    id: 0,
-                                    jsonrpc: "2.0",
-                                    method: "eth_call",
-                                    params: [
-                                      %{
-                                        data:
-                                          "0xc87b56dd000000000000000000000000000000000000000000000000fdd5b9fa9d4bfb20",
-                                        to: "0x5caebd3b32e210e85ce3e9d51638b9c445481567"
-                                      },
-                                      "latest"
-                                    ]
+    test "returns {:error, @no_uri_error} when empty uri is passed" do
+      error = {:error, "no uri"}
+      token_id = "TOKEN_ID"
+      hex_token_id = "HEX_TOKEN_ID"
+      from_base_uri = true
+
+      result = MetadataRetriever.fetch_json({:ok, [""]}, token_id, hex_token_id, from_base_uri)
+
+      assert result == error
+    end
+
+    test "returns {:error, @vm_execution_error} when 'execution reverted' error passed in uri" do
+      uri_error = {:error, "something happened: execution reverted"}
+      token_id = "TOKEN_ID"
+      hex_token_id = "HEX_TOKEN_ID"
+      from_base_uri = true
+      result_error = {:error, "VM execution error"}
+
+      result = MetadataRetriever.fetch_json(uri_error, token_id, hex_token_id, from_base_uri)
+
+      assert result == result_error
+    end
+
+    test "returns {:error, @vm_execution_error} when 'VM execution error' error passed in uri" do
+      error = {:error, "VM execution error"}
+      token_id = "TOKEN_ID"
+      hex_token_id = "HEX_TOKEN_ID"
+      from_base_uri = true
+
+      result = MetadataRetriever.fetch_json(error, token_id, hex_token_id, from_base_uri)
+
+      assert result == error
+    end
+
+    test "returns {:error, error} when all other errors passed in uri" do
+      error = {:error, "Some error"}
+      token_id = "TOKEN_ID"
+      hex_token_id = "HEX_TOKEN_ID"
+      from_base_uri = true
+
+      result = MetadataRetriever.fetch_json(error, token_id, hex_token_id, from_base_uri)
+
+      assert result == error
+    end
+
+    test "returns {:error, truncated_error} when long error passed in uri" do
+      error =
+        {:error,
+         "ERROR: Unable to establish a connection to the database server. The database server may be offline, or there could be a network issue preventing access. Please ensure that the database server is running and that the network configuration is correct. Additionally, check the database credentials and permissions to ensure they are valid. If the issue persists, contact your system administrator for further assistance. Error code: DB_CONN_FAILED_101234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"}
+
+      token_id = "TOKEN_ID"
+      hex_token_id = "HEX_TOKEN_ID"
+      from_base_uri = true
+
+      truncated_error =
+        {:error,
+         "ERROR: Unable to establish a connection to the database server. The database server may be offline, or there could be a network issue preventing access. Please ensure that the database server is running and that the network configuration is correct. Ad..."}
+
+      result = MetadataRetriever.fetch_json(error, token_id, hex_token_id, from_base_uri)
+
+      assert result == truncated_error
+    end
+
+    test "Constructs IPFS link with query param" do
+      configuration = Application.get_env(:indexer, :ipfs)
+
+      Application.put_env(:indexer, :ipfs,
+        gateway_url: Keyword.get(configuration, :gateway_url),
+        gateway_url_param_location: :query,
+        gateway_url_param_key: "x-apikey",
+        gateway_url_param_value: "mykey"
+      )
+
+      data = "QmT1Yz43R1PLn2RVovAnEM5dHQEvpTcnwgX8zftvY1FcjP"
+
+      result = %{
+        "name" => "asda",
+        "description" => "asda",
+        "salePrice" => 34,
+        "img_hash" => "QmUfW3PVnh9GGuHcQgc3ZeNEbhwp5HE8rS5ac9MDWWQebz",
+        "collectionId" => "1871_1665123820823"
       }
-                                ],
+
+      Application.put_env(:explorer, :http_adapter, Explorer.Mox.HTTPoison)
+
+      Explorer.Mox.HTTPoison
+      |> expect(:get, fn "https://ipfs.io/ipfs/QmT1Yz43R1PLn2RVovAnEM5dHQEvpTcnwgX8zftvY1FcjP?x-apikey=mykey",
+                         _headers,
                          _options ->
-          {:ok,
-           [
+        {:ok, %HTTPoison.Response{status_code: 200, body: Jason.encode!(result)}}
+      end)
+
+      assert {:ok,
               %{
-               id: 0,
-               jsonrpc: "2.0",
-               result:
-                 "0x0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003568747470733a2f2f7661756c742e7761727269646572732e636f6d2f31383239303732393934373636373130323439362e6a736f6e0000000000000000000000"
+                metadata: %{
+                  "collectionId" => "1871_1665123820823",
+                  "description" => "asda",
+                  "img_hash" => "QmUfW3PVnh9GGuHcQgc3ZeNEbhwp5HE8rS5ac9MDWWQebz",
+                  "name" => "asda",
+                  "salePrice" => 34
                 }
-           ]}
-        end)
+              }} == MetadataRetriever.fetch_json({:ok, [data]})
+
+      Application.put_env(:explorer, :http_adapter, HTTPoison)
+      Application.put_env(:indexer, :ipfs, configuration)
     end
 
-      assert %{
-               "c87b56dd" => {:ok, ["https://vault.warriders.com/18290729947667102496.json"]}
-             } ==
-               MetadataRetriever.query_contract(
-                 "0x5caebd3b32e210e85ce3e9d51638b9c445481567",
-                 %{
-                   "c87b56dd" => [18_290_729_947_667_102_496]
-                 },
-                 @abi
+    test "Constructs IPFS link with no query param, if gateway_url_param_location is invalid" do
+      configuration = Application.get_env(:indexer, :ipfs)
+
+      Application.put_env(:indexer, :ipfs,
+        gateway_url: Keyword.get(configuration, :gateway_url),
+        gateway_url_param_location: :query2,
+        gateway_url_param_key: "x-apikey",
+        gateway_url_param_value: "mykey"
       )
-    end
 
-    test "fetches json metadata for ERC-1155 token", %{json_rpc_named_arguments: json_rpc_named_arguments} do
-      if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do
-        EthereumJSONRPC.Mox
-        |> expect(:json_rpc, fn [
-                                  %{
-                                    id: 0,
-                                    jsonrpc: "2.0",
-                                    method: "eth_call",
-                                    params: [
-                                      %{
-                                        data:
-                                          "0x0e89341c000000000000000000000000000000000000000000000000fdd5b9fa9d4bfb20",
-                                        to: "0x5caebd3b32e210e85ce3e9d51638b9c445481567"
-                                      },
-                                      "latest"
-                                    ]
+      data = "QmT1Yz43R1PLn2RVovAnEM5dHQEvpTcnwgX8zftvY1FcjP"
+
+      result = %{
+        "name" => "asda",
+        "description" => "asda",
+        "salePrice" => 34,
+        "img_hash" => "QmUfW3PVnh9GGuHcQgc3ZeNEbhwp5HE8rS5ac9MDWWQebz",
+        "collectionId" => "1871_1665123820823"
       }
-                                ],
-                                _options ->
-          {:ok,
-           [
+
+      Application.put_env(:explorer, :http_adapter, Explorer.Mox.HTTPoison)
+
+      Explorer.Mox.HTTPoison
+      |> expect(:get, fn "https://ipfs.io/ipfs/QmT1Yz43R1PLn2RVovAnEM5dHQEvpTcnwgX8zftvY1FcjP", _headers, _options ->
+        {:ok, %HTTPoison.Response{status_code: 200, body: Jason.encode!(result)}}
+      end)
+
+      assert {:ok,
               %{
-               id: 0,
-               jsonrpc: "2.0",
-               result:
-                 "0x0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003568747470733a2f2f7661756c742e7761727269646572732e636f6d2f31383239303732393934373636373130323439362e6a736f6e0000000000000000000000"
+                metadata: %{
+                  "collectionId" => "1871_1665123820823",
+                  "description" => "asda",
+                  "img_hash" => "QmUfW3PVnh9GGuHcQgc3ZeNEbhwp5HE8rS5ac9MDWWQebz",
+                  "name" => "asda",
+                  "salePrice" => 34
                 }
-           ]}
-        end)
+              }} == MetadataRetriever.fetch_json({:ok, [data]})
+
+      Application.put_env(:explorer, :http_adapter, HTTPoison)
+      Application.put_env(:indexer, :ipfs, configuration)
     end
 
-      assert %{
-               "0e89341c" => {:ok, ["https://vault.warriders.com/18290729947667102496.json"]}
-             } ==
-               MetadataRetriever.query_contract(
-                 "0x5caebd3b32e210e85ce3e9d51638b9c445481567",
-                 %{
-                   "0e89341c" => [18_290_729_947_667_102_496]
-                 },
-                 @abi_uri
+    test "Constructs IPFS link with additional header" do
+      configuration = Application.get_env(:indexer, :ipfs)
+
+      Application.put_env(:indexer, :ipfs,
+        gateway_url: Keyword.get(configuration, :gateway_url),
+        gateway_url_param_location: :header,
+        gateway_url_param_key: "x-apikey",
+        gateway_url_param_value: "mykey"
       )
-    end
-  end
 
-  describe "fetch_json/1" do
-    setup do
-      bypass = Bypass.open()
+      data = "QmT1Yz43R1PLn2RVovAnEM5dHQEvpTcnwgX8zftvY1FcjP"
 
-      {:ok, bypass: bypass}
+      result = %{
+        "name" => "asda",
+        "description" => "asda",
+        "salePrice" => 34,
+        "img_hash" => "QmUfW3PVnh9GGuHcQgc3ZeNEbhwp5HE8rS5ac9MDWWQebz",
+        "collectionId" => "1871_1665123820823"
+      }
+
+      Application.put_env(:explorer, :http_adapter, Explorer.Mox.HTTPoison)
+
+      Explorer.Mox.HTTPoison
+      |> expect(:get, fn "https://ipfs.io/ipfs/QmT1Yz43R1PLn2RVovAnEM5dHQEvpTcnwgX8zftvY1FcjP",
+                         [{"x-apikey", "mykey"}],
+                         _options ->
+        {:ok, %HTTPoison.Response{status_code: 200, body: Jason.encode!(result)}}
+      end)
+
+      assert {:ok,
+              %{
+                metadata: %{
+                  "collectionId" => "1871_1665123820823",
+                  "description" => "asda",
+                  "img_hash" => "QmUfW3PVnh9GGuHcQgc3ZeNEbhwp5HE8rS5ac9MDWWQebz",
+                  "name" => "asda",
+                  "salePrice" => 34
+                }
+              }} == MetadataRetriever.fetch_json({:ok, [data]})
+
+      Application.put_env(:explorer, :http_adapter, HTTPoison)
+      Application.put_env(:indexer, :ipfs, configuration)
     end
 
     test "fetches json with latin1 encoding", %{bypass: bypass} do
@@ -190,7 +250,8 @@ defmodule Explorer.Token.InstanceMetadataRetrieverTest do
         Conn.resp(conn, 200, json)
       end)
 
-      {:ok, %{metadata: metadata}} = MetadataRetriever.fetch_metadata_from_uri("http://localhost:#{bypass.port}#{path}")
+      {:ok, %{metadata: metadata}} =
+        MetadataRetriever.fetch_metadata_from_uri("http://localhost:#{bypass.port}#{path}", [])
 
       assert Map.get(metadata, "attributes") == Jason.decode!(attributes)
     end

config/config_helper.exs

@@ -1,4 +1,6 @@
 defmodule ConfigHelper do
+  require Logger
+
   import Bitwise
   alias Explorer.ExchangeRates.Source
   alias Explorer.Market.History.Source.{MarketCap, Price, TVL}
@@ -95,6 +97,35 @@ defmodule ConfigHelper do
     end
   end
 
+  @doc """
+  Parses value of env var through catalogued values list. If a value is not in the list, nil is returned.
+  Also, the application shutdown option is supported, if a value is wrong.
+  """
+  @spec parse_catalog_value(String.t(), List.t(), bool(), String.t() | nil) :: atom() | nil
+  def parse_catalog_value(env_var, catalog, shutdown_on_wrong_value?, default_value \\ nil) do
+    value = env_var |> safe_get_env(default_value)
+
+    if value !== "" do
+      if value in catalog do
+        String.to_atom(value)
+      else
+        if shutdown_on_wrong_value? do
+          Logger.error(wrong_value_error(value, env_var, catalog))
+          exit(:shutdown)
+        else
+          Logger.warning(wrong_value_error(value, env_var, catalog))
+          nil
+        end
+      end
+    else
+      nil
+    end
+  end
+
+  defp wrong_value_error(value, env_var, catalog) do
+    "Wrong value #{value} of #{env_var} environment variable. Supported values are #{inspect(catalog)}"
+  end
+
   def safe_get_env(env_var, default_value) do
     env_var
     |> System.get_env(default_value)

config/runtime.exs

@@ -598,9 +598,15 @@ config :indexer,
     ConfigHelper.parse_integer_env_var("INDEXER_TOKEN_BALANCES_FETCHER_INIT_QUERY_LIMIT", 100_000),
   coin_balances_fetcher_init_limit:
     ConfigHelper.parse_integer_env_var("INDEXER_COIN_BALANCES_FETCHER_INIT_QUERY_LIMIT", 2000),
-  ipfs_gateway_url: System.get_env("IPFS_GATEWAY_URL", "https://ipfs.io/ipfs"),
   graceful_shutdown_period: ConfigHelper.parse_time_env_var("INDEXER_GRACEFUL_SHUTDOWN_PERIOD", "5m")
 
+config :indexer, :ipfs,
+  gateway_url: System.get_env("IPFS_GATEWAY_URL", "https://ipfs.io/ipfs"),
+  gateway_url_param_key: System.get_env("IPFS_GATEWAY_URL_PARAM_KEY"),
+  gateway_url_param_value: System.get_env("IPFS_GATEWAY_URL_PARAM_VALUE"),
+  gateway_url_param_location:
+    ConfigHelper.parse_catalog_value("IPFS_GATEWAY_URL_PARAM_LOCATION", ["query", "header"], true)
+
 config :indexer, Indexer.Supervisor, enabled: !ConfigHelper.parse_bool_env_var("DISABLE_INDEXER")
 
 config :indexer, Indexer.Fetcher.TransactionAction.Supervisor,

cspell.json

@@ -368,6 +368,7 @@
         "munknownc",
         "munknowne",
         "mydep",
+        "mykey",
         "nanomorph",
         "nbsp",
         "newkey",