Extend token instance metadata parsing functionality

CHANGELOG.md

@@ -7,7 +7,7 @@
 ### Fixes
 
 - [#7008](https://github.com/blockscout/blockscout/pull/7008) - Fetch image/video content from IPFS link
-- [#7007](https://github.com/blockscout/blockscout/pull/7007) - Token instance fetcher fixes
+- [#7007](https://github.com/blockscout/blockscout/pull/7007), [#7031](https://github.com/blockscout/blockscout/pull/7031) - Token instance fetcher fixes
 - [#7009](https://github.com/blockscout/blockscout/pull/7009) - Fix updating coin balances with empty value
 
 ### Chore

apps/block_scout_web/lib/block_scout_web/views/nft_helpers.ex

@@ -2,6 +2,8 @@ defmodule BlockScoutWeb.NFTHelpers do
   @moduledoc """
     Module with functions for NFT view
   """
+  @ipfs_protocol "ipfs://"
+
   def get_media_src(nil, _), do: nil
 
   def get_media_src(metadata, high_quality_media?) do
@@ -56,18 +58,29 @@ defmodule BlockScoutWeb.NFTHelpers do
     |> compose_ipfs_url()
   end
 
+  def compose_ipfs_url(nil), do: nil
+
   def compose_ipfs_url(image_url) do
+    image_url_downcase =
+      image_url
+      |> String.downcase()
+
     cond do
-      image_url =~ ~r/^ipfs:\/\/ipfs/ ->
-        "ipfs://ipfs" <> ipfs_uid = image_url
-        "https://ipfs.io/ipfs/" <> ipfs_uid
+      image_url_downcase =~ ~r/^ipfs:\/\/ipfs/ ->
+        prefix = @ipfs_protocol <> "ipfs/"
+        ipfs_link(image_url, prefix)
 
-      image_url =~ ~r/^ipfs:\/\// ->
-        "ipfs://" <> ipfs_uid = image_url
-        "https://ipfs.io/ipfs/" <> ipfs_uid
+      image_url_downcase =~ ~r/^ipfs:\/\// ->
+        prefix = @ipfs_protocol
+        ipfs_link(image_url, prefix)
 
       true ->
         image_url
     end
   end
+
+  defp ipfs_link(image_url, prefix) do
+    ipfs_uid = String.slice(image_url, String.length(prefix)..-1)
+    "https://ipfs.io/ipfs/" <> ipfs_uid
+  end
 end

apps/block_scout_web/test/block_scout_web/views/nft_helpers_test.exs

@@ -0,0 +1,28 @@
+defmodule BlockScoutWeb.NFTHelpersTest do
+  use BlockScoutWeb.ConnCase, async: true
+
+  alias BlockScoutWeb.{NFTHelpers}
+
+  describe "compose_ipfs_url/1" do
+    test "transforms ipfs link like ipfs://${id}" do
+      url = "ipfs://QmYFf7D2UtqnNz8Lu57Gnk3dxgdAiuboPWMEaNNjhr29tS/hidden.png"
+
+      assert "https://ipfs.io/ipfs/QmYFf7D2UtqnNz8Lu57Gnk3dxgdAiuboPWMEaNNjhr29tS/hidden.png" ==
+               BlockScoutWeb.NFTHelpers.compose_ipfs_url(url)
+    end
+
+    test "transforms ipfs link like ipfs://ipfs" do
+      url = "ipfs://ipfs/Qmbgk4Ps5kiVdeYCHufMFgqzWLFuovFRtenY5P8m9vr9XW/animation.mp4"
+
+      assert "https://ipfs.io/ipfs/Qmbgk4Ps5kiVdeYCHufMFgqzWLFuovFRtenY5P8m9vr9XW/animation.mp4" ==
+               BlockScoutWeb.NFTHelpers.compose_ipfs_url(url)
+    end
+
+    test "transforms ipfs link in different case" do
+      url = "IpFs://baFybeid4ed2ua7fwupv4nx2ziczr3edhygl7ws3yx6y2juon7xakgj6cfm/51.json"
+
+      assert "https://ipfs.io/ipfs/baFybeid4ed2ua7fwupv4nx2ziczr3edhygl7ws3yx6y2juon7xakgj6cfm/51.json" ==
+               BlockScoutWeb.NFTHelpers.compose_ipfs_url(url)
+    end
+  end
+end

apps/explorer/lib/explorer/token/instance_metadata_retriever.ex

@@ -59,6 +59,8 @@ defmodule Explorer.Token.InstanceMetadataRetriever do
 
   @no_uri_error "no uri"
   @vm_execution_error "VM execution error"
+  @ipfs_protocol "ipfs://"
+  @ipfs_link "https://ipfs.io/ipfs/"
 
   # https://eips.ethereum.org/EIPS/eip-1155#metadata
   @erc1155_token_id_placeholder "{id}"
@@ -100,65 +102,64 @@ defmodule Explorer.Token.InstanceMetadataRetriever do
     {:ok, %{error: @no_uri_error}}
   end
 
-  def fetch_json(uri, _hex_token_id)
-      when uri in [
-             %{@token_uri => {:error, "(-32015) VM execution error."}},
-             %{@uri => {:error, "(-32015) VM execution error."}},
-             %{@token_uri => {:error, "(-32000) execution reverted"}},
-             %{@uri => {:error, "(-32000) execution reverted"}}
-           ] do
-    {:ok, %{error: @vm_execution_error}}
+  def fetch_json(%{@token_uri => uri}, hex_token_id) do
+    fetch_json_from_uri(uri, hex_token_id)
   end
 
-  def fetch_json(%{@token_uri => {:error, "(-32015) VM execution error." <> _}}, _hex_token_id) do
-    {:ok, %{error: @vm_execution_error}}
+  def fetch_json(%{@uri => uri}, hex_token_id) do
+    fetch_json_from_uri(uri, hex_token_id)
   end
 
-  def fetch_json(%{@uri => {:error, "(-32015) VM execution error." <> _}}, _hex_token_id) do
-    {:ok, %{error: @vm_execution_error}}
+  # CIDv0 IPFS links # https://docs.ipfs.tech/concepts/content-addressing/#version-0-v0
+  def fetch_json("Qm" <> _ = result, hex_token_id) do
+    if String.length(result) == 46 do
+      fetch_json_from_uri({:ok, [@ipfs_link <> result]}, hex_token_id)
+    else
+      Logger.debug(["Unknown metadata format result #{inspect(result)}."], fetcher: :token_instances)
+
+      {:error, result}
+    end
   end
 
-  def fetch_json(%{@token_uri => {:error, "(-32000) execution reverted" <> _}}, _hex_token_id) do
-    {:ok, %{error: @vm_execution_error}}
+  def fetch_json(result, hex_token_id) do
+    case URI.parse(result) do
+      %URI{host: nil} ->
+        Logger.debug(["Unknown metadata format #{inspect(result)}."], fetcher: :token_instances)
+
+        {:error, result}
+
+      _ ->
+        fetch_json_from_uri({:ok, [result]}, hex_token_id)
+    end
   end
 
-  def fetch_json(%{@uri => {:error, "(-32000) execution reverted" <> _}}, _hex_token_id) do
+  defp fetch_json_from_uri({:error, error}, _hex_token_id) do
+    if error =~ "execution reverted" or error =~ @vm_execution_error do
       {:ok, %{error: @vm_execution_error}}
-  end
+    else
+      Logger.debug(["Unknown metadata format error #{inspect(error)}."], fetcher: :token_instances)
 
-  def fetch_json(%{@token_uri => {:ok, ["http://" <> _ = token_uri]}}, hex_token_id) do
-    fetch_metadata_inner(token_uri, hex_token_id)
+      {:error, error}
     end
-
-  def fetch_json(%{@uri => {:ok, ["http://" <> _ = token_uri]}}, hex_token_id) do
-    fetch_metadata_inner(token_uri, hex_token_id)
   end
 
-  def fetch_json(%{@token_uri => {:ok, ["https://" <> _ = token_uri]}}, hex_token_id) do
+  defp fetch_json_from_uri({:ok, ["'" <> token_uri]}, hex_token_id) do
+    token_uri = token_uri |> String.split("'") |> List.first()
     fetch_metadata_inner(token_uri, hex_token_id)
   end
 
-  def fetch_json(%{@uri => {:ok, ["https://" <> _ = token_uri]}}, hex_token_id) do
+  defp fetch_json_from_uri({:ok, ["http://" <> _ = token_uri]}, hex_token_id) do
     fetch_metadata_inner(token_uri, hex_token_id)
   end
 
-  def fetch_json(%{@token_uri => {:ok, ["data:application/json," <> json]}}, hex_token_id) do
-    decoded_json = URI.decode(json)
-
-    fetch_json(%{@token_uri => {:ok, [decoded_json]}}, hex_token_id)
-  rescue
-    e ->
-      Logger.debug(["Unknown metadata format #{inspect(json)}.", Exception.format(:error, e, __STACKTRACE__)],
-        fetcher: :token_instances
-      )
-
-      {:error, json}
+  defp fetch_json_from_uri({:ok, ["https://" <> _ = token_uri]}, hex_token_id) do
+    fetch_metadata_inner(token_uri, hex_token_id)
   end
 
-  def fetch_json(%{@uri => {:ok, ["data:application/json," <> json]}}, hex_token_id) do
+  defp fetch_json_from_uri({:ok, ["data:application/json," <> json]}, hex_token_id) do
     decoded_json = URI.decode(json)
 
-    fetch_json(%{@uri => {:ok, [decoded_json]}}, hex_token_id)
+    fetch_json_from_uri({:ok, [decoded_json]}, hex_token_id)
   rescue
     e ->
       Logger.debug(["Unknown metadata format #{inspect(json)}.", Exception.format(:error, e, __STACKTRACE__)],
@@ -168,10 +169,10 @@ defmodule Explorer.Token.InstanceMetadataRetriever do
       {:error, json}
   end
 
-  def fetch_json(%{@token_uri => {:ok, ["data:application/json;base64," <> base64_encoded_json]}}, hex_token_id) do
+  defp fetch_json_from_uri({:ok, ["data:application/json;base64," <> base64_encoded_json]}, hex_token_id) do
     case Base.decode64(base64_encoded_json) do
       {:ok, base64_decoded} ->
-        fetch_json(%{@token_uri => {:ok, [base64_decoded]}}, hex_token_id)
+        fetch_json_from_uri({:ok, [base64_decoded]}, hex_token_id)
 
       _ ->
         {:error, base64_encoded_json}
@@ -189,54 +190,15 @@ defmodule Explorer.Token.InstanceMetadataRetriever do
       {:error, base64_encoded_json}
   end
 
-  def fetch_json(%{@uri => {:ok, ["data:application/json;base64," <> base64_encoded_json]}}, hex_token_id) do
-    case Base.decode64(base64_encoded_json) do
-      {:ok, base64_decoded} ->
-        fetch_json(%{@uri => {:ok, [base64_decoded]}}, hex_token_id)
-
-      _ ->
-        {:error, base64_encoded_json}
-    end
-  rescue
-    e ->
-      Logger.debug(
-        ["Unknown metadata format base64 #{inspect(base64_encoded_json)}", Exception.format(:error, e, __STACKTRACE__)],
-        fetcher: :token_instances
-      )
-
-      {:error, base64_encoded_json}
-  end
-
-  def fetch_json(%{@token_uri => {:ok, ["ipfs://ipfs/" <> right]}}, hex_token_id) do
-    fetch_from_ipfs(right, hex_token_id)
-  end
-
-  def fetch_json(%{@uri => {:ok, ["ipfs://ipfs/" <> right]}}, hex_token_id) do
-    fetch_from_ipfs(right, hex_token_id)
-  end
-
-  def fetch_json(%{@token_uri => {:ok, ["ipfs://" <> right]}}, hex_token_id) do
+  defp fetch_json_from_uri({:ok, ["#{@ipfs_protocol}ipfs/" <> right]}, hex_token_id) do
     fetch_from_ipfs(right, hex_token_id)
   end
 
-  def fetch_json(%{@uri => {:ok, ["ipfs://" <> right]}}, hex_token_id) do
+  defp fetch_json_from_uri({:ok, [@ipfs_protocol <> right]}, hex_token_id) do
     fetch_from_ipfs(right, hex_token_id)
   end
 
-  def fetch_json(%{@token_uri => {:ok, [json]}}, hex_token_id) do
-    {:ok, json} = decode_json(json)
-
-    check_type(json, hex_token_id)
-  rescue
-    e ->
-      Logger.debug(["Unknown metadata format #{inspect(json)}.", Exception.format(:error, e, __STACKTRACE__)],
-        fetcher: :token_instances
-      )
-
-      {:error, json}
-  end
-
-  def fetch_json(%{@uri => {:ok, [json]}}, hex_token_id) do
+  defp fetch_json_from_uri({:ok, [json]}, hex_token_id) do
     {:ok, json} = decode_json(json)
 
     check_type(json, hex_token_id)
@@ -249,14 +211,14 @@ defmodule Explorer.Token.InstanceMetadataRetriever do
       {:error, json}
   end
 
-  def fetch_json(result, _hex_token_id) do
-    Logger.debug(["Unknown metadata format #{inspect(result)}."], fetcher: :token_instances)
+  defp fetch_json_from_uri(uri, _hex_token_id) do
+    Logger.debug(["Unknown metadata uri format #{inspect(uri)}."], fetcher: :token_instances)
 
-    {:error, result}
+    {:error, uri}
   end
 
   defp fetch_from_ipfs(ipfs_uid, hex_token_id) do
-    ipfs_url = "https://ipfs.io/ipfs/" <> ipfs_uid
+    ipfs_url = @ipfs_link <> ipfs_uid
     fetch_metadata_inner(ipfs_url, hex_token_id)
   end
 

apps/explorer/test/explorer/token/instance_metadata_retriever_test.exs

@@ -361,7 +361,7 @@ defmodule Explorer.Token.InstanceMetadataRetrieverTest do
                 metadata: %{
                   "image" => "https://ipfs.io/ipfs/bafybeig6nlmyzui7llhauc52j2xo5hoy4lzp6442lkve5wysdvjkizxonu"
                 }
-              }} = InstanceMetadataRetriever.fetch_json(data)
+              }} == InstanceMetadataRetriever.fetch_json(data)
     end
 
     test "Fetches metadata from ipfs" do
@@ -373,16 +373,84 @@ defmodule Explorer.Token.InstanceMetadataRetrieverTest do
            ]}
       }
 
+      {:ok,
+       %{
+         metadata: metadata
+       }} = InstanceMetadataRetriever.fetch_json(data)
+
+      assert "ipfs://bafybeihxuj3gxk7x5p36amzootyukbugmx3pw7dyntsrohg3se64efkuga/51.png" == Map.get(metadata, "image")
+    end
+
+    test "Fetches metadata from '${url}'" do
+      data = %{
+        "c87b56dd" =>
+          {:ok,
+           [
+             "'https://cards.collecttrumpcards.com/data/8/8578.json'"
+           ]}
+      }
+
       assert {:ok,
               %{
                 metadata: %{
-                  "image" => "ipfs://bafybeihxuj3gxk7x5p36amzootyukbugmx3pw7dyntsrohg3se64efkuga/51.png",
-                  "attributes" => _,
-                  "description" => "No roadmap Just OP NOK...But This NFT can use in Sobta ecosystem (if any)",
-                  "edition" => 51,
-                  "name" => "SobtaOpGenesis #51"
+                  "attributes" => [
+                    %{"trait_type" => "Character", "value" => "Blue Suit Boxing Glove"},
+                    %{"trait_type" => "Face", "value" => "Wink"},
+                    %{"trait_type" => "Hat", "value" => "Blue"},
+                    %{"trait_type" => "Background", "value" => "Red Carpet"}
+                  ],
+                  "image" => "https://cards.collecttrumpcards.com/cards/0c68b1ab6.jpg",
+                  "name" => "Trump Digital Trading Card #8578",
+                  "tokenId" => 8578
                 }
-              }} = InstanceMetadataRetriever.fetch_json(data)
+              }} == InstanceMetadataRetriever.fetch_json(data)
+    end
+
+    test "Process custom execution reverted" do
+      data = %{
+        "c87b56dd" =>
+          {:error,
+           "(3) execution reverted: Nonexistent token (0x08c379a0000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000114e6f6e6578697374656e7420746f6b656e000000000000000000000000000000)"}
+      }
+
+      assert {:ok, %{error: "VM execution error"}} == InstanceMetadataRetriever.fetch_json(data)
+    end
+
+    test "Process CIDv0 IPFS links" do
+      data = "QmT1Yz43R1PLn2RVovAnEM5dHQEvpTcnwgX8zftvY1FcjP"
+
+      assert {:ok,
+              %{
+                metadata: %{
+                  "collectionId" => "1871_1665123820823",
+                  "description" => "asda",
+                  "img_hash" => "QmUfW3PVnh9GGuHcQgc3ZeNEbhwp5HE8rS5ac9MDWWQebz",
+                  "name" => "asda",
+                  "salePrice" => 34
+                }
+              }} == InstanceMetadataRetriever.fetch_json(data)
+    end
+
+    test "Process URI directly from link" do
+      data = "https://dejob.io/api/dejobio/v1/nftproduct/1"
+
+      assert {:ok,
+              %{
+                metadata: %{
+                  "description" =>
+                    "\\\"Blue Reign: The Dragon Football Champion of the Floral City\\\" is a science fiction story about a dragon who loves playing football and dreams of becoming a champion. The story takes place in a futuristic city full of flowers and blue light, and it is raining throughout the story.\r\n\r\nThroughout the story, the dragon faces challenges on and off the field, including intense training regimens, rival teams, and personal struggles. He perseveres through these obstacles and incorporates new techniques and strategies into his gameplay.\r\n\r\nAs the playoffs approach, the dragon\\'s team faces increasingly tough opponents, culminating in a highly anticipated championship game against their long-standing rivals, the Storm Hawks. The dragon\\'s heart-pumping performance and his team\\'s impressive plays lead them to victory, and they celebrate their status as champions.\r\n\r\nThe story ultimately focuses on the dragon\\'s journey towards achieving his dream and the teamwork and dedication required to succeed in a highly competitive sport.",
+                  "name" => "Blue Reign: The Dragon Football Champion of the Floral City",
+                  "attributes" => [
+                    %{"trait_type" => "Product Type", "value" => "Book"},
+                    %{"display_type" => "number", "trait_type" => "Total Sold", "value" => "0"},
+                    %{"display_type" => "number", "trait_type" => "Success Sold", "value" => "0"},
+                    %{"max_value" => "100", "trait_type" => "Success Rate", "value" => "0"}
+                  ],
+                  "external_url" => "https://dejob.io/?p=49",
+                  "image" =>
+                    "https://cdn.discordapp.com/attachments/1008567215739650078/1080111780858187796/savechives_a_dragon_playing_football_in_a_city_full_of_flowers__0739cc42-aae1-4909-a964-3f9c0ed1a9ed.png"
+                }
+              }} == InstanceMetadataRetriever.fetch_json(data)
     end
   end
 end