Merge pull request #2779 from poanetwork/ab-fix-token-instance-issues

fix fetching `latin1` encoded data

CHANGELOG.md

@@ -18,6 +18,7 @@
 - [#2470](https://github.com/poanetwork/blockscout/pull/2470) - Allow Realtime Fetcher to wait for small skips
 
 ### Fixes
+- [#2779](https://github.com/poanetwork/blockscout/pull/2779) - fix fetching `latin1` encoded data
 - [#2783](https://github.com/poanetwork/blockscout/pull/2783) - Fix stuck value and ticker on the token page
 - [#2781](https://github.com/poanetwork/blockscout/pull/2781) - optimize txlist json rpc
 - [#2770](https://github.com/poanetwork/blockscout/pull/2770) - do not re-fetch token instances without uris

apps/explorer/config/dev.exs

@@ -18,6 +18,11 @@ config :logger, :reading_token_functions,
   path: Path.absname("logs/dev/explorer/tokens/reading_functions.log"),
   metadata_filter: [fetcher: :token_functions]
 
+config :logger, :token_instances,
+  level: :debug,
+  path: Path.absname("logs/dev/explorer/tokens/token_instances.log"),
+  metadata_filter: [fetcher: :token_instances]
+
 import_config "dev.secret.exs"
 
 variant =

apps/explorer/config/prod.exs

@@ -21,6 +21,12 @@ config :logger, :reading_token_functions,
   metadata_filter: [fetcher: :token_functions],
   rotate: %{max_bytes: 52_428_800, keep: 19}
 
+config :logger, :token_instances,
+  level: :debug,
+  path: Path.absname("logs/prod/explorer/tokens/token_instances.log"),
+  metadata_filter: [fetcher: :token_instances],
+  rotate: %{max_bytes: 52_428_800, keep: 19}
+
 variant =
   if is_nil(System.get_env("ETHEREUM_JSONRPC_VARIANT")) do
     "parity"

apps/explorer/lib/explorer/token/instance_metadata_retriever.ex

@@ -48,33 +48,50 @@ defmodule Explorer.Token.InstanceMetadataRetriever do
     Reader.query_contract(contract_address_hash, @abi, contract_functions)
   end
 
-  defp fetch_json(%{"tokenURI" => {:ok, [""]}}) do
+  def fetch_json(%{"tokenURI" => {:ok, [""]}}) do
     {:ok, %{error: @no_uri_error}}
   end
 
-  defp fetch_json(%{"tokenURI" => {:error, "(-32015) VM execution error."}}) do
+  def fetch_json(%{"tokenURI" => {:error, "(-32015) VM execution error."}}) do
     {:ok, %{error: @no_uri_error}}
   end
 
-  defp fetch_json(%{"tokenURI" => {:ok, ["http://" <> _ = token_uri]}}) do
+  def fetch_json(%{"tokenURI" => {:ok, ["http://" <> _ = token_uri]}}) do
     fetch_metadata(token_uri)
   end
 
-  defp fetch_json(%{"tokenURI" => {:ok, ["https://" <> _ = token_uri]}}) do
+  def fetch_json(%{"tokenURI" => {:ok, ["https://" <> _ = token_uri]}}) do
     fetch_metadata(token_uri)
   end
 
-  defp fetch_json(%{"tokenURI" => {:ok, [json]}}) do
+  def fetch_json(%{"tokenURI" => {:ok, ["data:application/json," <> json]}}) do
-    Jason.decode(json)
+    decoded_json = URI.decode(json)
+
+    fetch_json(%{"tokenURI" => {:ok, [decoded_json]}})
+  rescue
+    e ->
+      Logger.debug(["Unknown metadata format #{inspect(json)}. error #{inspect(e)}"],
+        fetcher: :token_instances
+      )
+
+      {:error, json}
+  end
+
+  def fetch_json(%{"tokenURI" => {:ok, [json]}}) do
+    {:ok, json} = decode_json(json)
+
+    {:ok, %{metadata: json}}
   rescue
     e ->
-      Logger.error(fn -> ["Unknown metadata format #{inspect(json)}. error #{inspect(e)}"] end)
+      Logger.debug(["Unknown metadata format #{inspect(json)}. error #{inspect(e)}"],
+        fetcher: :token_instances
+      )
 
       {:error, json}
   end
 
-  defp fetch_json(result) do
+  def fetch_json(result) do
-    Logger.error(fn -> ["Unknown metadata format #{inspect(result)}."] end)
+    Logger.debug(["Unknown metadata format #{inspect(result)}."], fetcher: :token_instances)
 
     {:error, result}
   end
@@ -82,7 +99,7 @@ defmodule Explorer.Token.InstanceMetadataRetriever do
   defp fetch_metadata(token_uri) do
     case HTTPoison.get(token_uri) do
       {:ok, %Response{body: body, status_code: 200}} ->
-        {:ok, json} = Jason.decode(body)
+        {:ok, json} = decode_json(body)
 
         {:ok, %{metadata: json}}
 
@@ -94,8 +111,20 @@ defmodule Explorer.Token.InstanceMetadataRetriever do
     end
   rescue
     e ->
-      Logger.error(fn -> ["Could not send request to token uri #{inspect(token_uri)}. error #{inspect(e)}"] end)
+      Logger.debug(["Could not send request to token uri #{inspect(token_uri)}. error #{inspect(e)}"],
+        fetcher: :token_instances
+      )
 
       {:error, :request_error}
   end
+
+  defp decode_json(body) do
+    if String.valid?(body) do
+      Jason.decode(body)
+    else
+      body
+      |> :unicode.characters_to_binary(:latin1)
+      |> Jason.decode()
+    end
+  end
 end

apps/explorer/test/explorer/token/instance_metadata_retriever_test.exs

@@ -2,6 +2,7 @@ defmodule Explorer.Token.InstanceMetadataRetrieverTest do
   use EthereumJSONRPC.Case
 
   alias Explorer.Token.InstanceMetadataRetriever
+  alias Plug.Conn
 
   import Mox
 
@@ -50,4 +51,51 @@ defmodule Explorer.Token.InstanceMetadataRetrieverTest do
                })
     end
   end
+
+  describe "fetch_json/1" do
+    setup do
+      bypass = Bypass.open()
+
+      {:ok, bypass: bypass}
+    end
+
+    test "fetches json with latin1 encoding", %{bypass: bypass} do
+      json = """
+      {
+        "name": "Sérgio Mendonça"
+      }
+      """
+
+      Bypass.expect(bypass, "GET", "/api/card/55265", fn conn ->
+        Conn.resp(conn, 200, json)
+      end)
+
+      assert {:ok, %{metadata: %{"name" => "Sérgio Mendonça"}}} ==
+               InstanceMetadataRetriever.fetch_json(%{
+                 "tokenURI" => {:ok, ["http://localhost:#{bypass.port}/api/card/55265"]}
+               })
+    end
+
+    test "decodes json file in tokenURI" do
+      data = %{
+        "tokenURI" =>
+          {:ok,
+           [
+             "data:application/json,{\"name\":\"Home%20Address%20-%200x0000000000C1A6066c6c8B9d63e9B6E8865dC117\",\"description\":\"This%20NFT%20can%20be%20redeemed%20on%20HomeWork%20to%20grant%20a%20controller%20the%20exclusive%20right%20to%20deploy%20contracts%20with%20arbitrary%20bytecode%20to%20the%20designated%20home%20address.\",\"image\":\"data:image/svg+xml;charset=utf-8;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAxNDQgNzIiPjxzdHlsZT48IVtDREFUQVsuQntzdHJva2UtbGluZWpvaW46cm91bmR9LkN7c3Ryb2tlLW1pdGVybGltaXQ6MTB9LkR7c3Ryb2tlLXdpZHRoOjJ9LkV7ZmlsbDojOWI5YjlhfS5Ge3N0cm9rZS1saW5lY2FwOnJvdW5kfV1dPjwvc3R5bGU+PGcgdHJhbnNmb3JtPSJtYXRyaXgoMS4wMiAwIDAgMS4wMiA4LjEgMCkiPjxwYXRoIGZpbGw9IiNmZmYiIGQ9Ik0xOSAzMmgzNHYyNEgxOXoiLz48ZyBzdHJva2U9IiMwMDAiIGNsYXNzPSJCIEMgRCI+PHBhdGggZmlsbD0iI2E1NzkzOSIgZD0iTTI1IDQwaDl2MTZoLTl6Ii8+PHBhdGggZmlsbD0iIzkyZDNmNSIgZD0iTTQwIDQwaDh2N2gtOHoiLz48cGF0aCBmaWxsPSIjZWE1YTQ3IiBkPSJNNTMgMzJIMTl2LTFsMTYtMTYgMTggMTZ6Ii8+PHBhdGggZmlsbD0ibm9uZSIgZD0iTTE5IDMyaDM0djI0SDE5eiIvPjxwYXRoIGZpbGw9IiNlYTVhNDciIGQ9Ik0yOSAyMWwtNSA1di05aDV6Ii8+PC9nPjwvZz48ZyB0cmFuc2Zvcm09Im1hdHJpeCguODQgMCAwIC44NCA2NSA1KSI+PHBhdGggZD0iTTkuNSAyMi45bDQuOCA2LjRhMy4xMiAzLjEyIDAgMCAxLTMgMi4ybC00LjgtNi40Yy4zLTEuNCAxLjYtMi40IDMtMi4yeiIgZmlsbD0iI2QwY2ZjZSIvPjxwYXRoIGZpbGw9IiMwMTAxMDEiIGQ9Ik00MS43IDM4LjVsNS4xLTYuNSIvPjxwYXRoIGQ9Ik00Mi45IDI3LjhMMTguNCA1OC4xIDI0IDYybDIxLjgtMjcuMyAyLjMtMi44eiIgY2xhc3M9IkUiLz48cGF0aCBmaWxsPSIjMDEwMTAxIiBkPSJNNDMuNCAyOS4zbC00LjcgNS44Ii8+PHBhdGggZD0iTTQ2LjggMzJjMy4yIDIuNiA4LjcgMS4yIDEyLjEtMy4yczMuNi05LjkuMy0xMi41bC01LjEgNi41LTIuOC0uMS0uNy0yLjcgNS4xLTYuNWMtMy4yLTIuNi04LjctMS4yLTEyLjEgMy4ycy0zLjYgOS45LS4zIDEyLjUiIGNsYXNzPSJFIi8+PHBhdGggZmlsbD0iI2E1NzkzOSIgZD0iTTI3LjMgMjZsMTEuOCAxNS43IDMuNCAyLjQgOS4xIDE0LjQtMy4yIDIuMy0xIC43LTEwLjItMTMuNi0xLjMtMy45LTExLjgtMTUuN3oiLz48cGF0aCBkPSJNMTIgMTkuOWw1LjkgNy45IDEwLjItNy42LTMuNC00LjVzNi44LTUuMSAxMC43LTQuNWMwIDAtNi42LTMtMTMuMyAxLjFTMTIgMTkuOSAxMiAxOS45eiIgY2xhc3M9IkUiLz48ZyBmaWxsPSJub25lIiBzdHJva2U9IiMwMDAiIGNsYXNzPSJCIEMgRCI+PHBhdGggZD0iTTUyIDU4LjlMNDAuOSA0My4ybC0zLjEtMi4zLTEwLjYtMTQuNy0yLjkgMi4yIDEwLjYgMTQuNyAxLjEgMy42IDExLjUgMTUuNXpNMTIuNSAxOS44bDUuOCA4IDEwLjMtNy40LTMuMy00LjZzNi45LTUgMTAuOC00LjNjMCAwLTYuNi0zLjEtMTMuMy45cy0xMC4zIDcuNC0xMC4zIDcuNHptLTIuNiAyLjlsNC43IDYuNWMtLjUgMS4zLTEuNyAyLjEtMyAyLjJsLTQuNy02LjVjLjMtMS40IDEuNi0yLjQgMy0yLjJ6Ii8+PHBhdGggZD0iTTQxLjMgMzguNWw1LjEtNi41bS0zLjUtMi43bC00LjYgNS44bTguMS0zLjFjMy4yIDIuNiA4LjcgMS4yIDEyLjEtMy4yczMuNi05LjkuMy0xMi41bC01LjEgNi41LTIuOC0uMS0uOC0yLjcgNS4xLTYuNWMtMy4yLTIuNi04LjctMS4yLTEyLjEgMy4yLTMuNCA0LjMtMy42IDkuOS0uMyAxMi41IiBjbGFzcz0iRiIvPjxwYXRoIGQ9Ik0zMC44IDQ0LjRMMTkgNTguOWw0IDMgMTAtMTIuNyIgY2xhc3M9IkYiLz48L2c+PC9nPjwvc3ZnPg==\"}"
+           ]}
+      }
+
+      assert InstanceMetadataRetriever.fetch_json(data) ==
+               {:ok,
+                %{
+                  metadata: %{
+                    "description" =>
+                      "This NFT can be redeemed on HomeWork to grant a controller the exclusive right to deploy contracts with arbitrary bytecode to the designated home address.",
+                    "image" =>
+                      "data:image/svg+xml;charset=utf-8;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAxNDQgNzIiPjxzdHlsZT48IVtDREFUQVsuQntzdHJva2UtbGluZWpvaW46cm91bmR9LkN7c3Ryb2tlLW1pdGVybGltaXQ6MTB9LkR7c3Ryb2tlLXdpZHRoOjJ9LkV7ZmlsbDojOWI5YjlhfS5Ge3N0cm9rZS1saW5lY2FwOnJvdW5kfV1dPjwvc3R5bGU+PGcgdHJhbnNmb3JtPSJtYXRyaXgoMS4wMiAwIDAgMS4wMiA4LjEgMCkiPjxwYXRoIGZpbGw9IiNmZmYiIGQ9Ik0xOSAzMmgzNHYyNEgxOXoiLz48ZyBzdHJva2U9IiMwMDAiIGNsYXNzPSJCIEMgRCI+PHBhdGggZmlsbD0iI2E1NzkzOSIgZD0iTTI1IDQwaDl2MTZoLTl6Ii8+PHBhdGggZmlsbD0iIzkyZDNmNSIgZD0iTTQwIDQwaDh2N2gtOHoiLz48cGF0aCBmaWxsPSIjZWE1YTQ3IiBkPSJNNTMgMzJIMTl2LTFsMTYtMTYgMTggMTZ6Ii8+PHBhdGggZmlsbD0ibm9uZSIgZD0iTTE5IDMyaDM0djI0SDE5eiIvPjxwYXRoIGZpbGw9IiNlYTVhNDciIGQ9Ik0yOSAyMWwtNSA1di05aDV6Ii8+PC9nPjwvZz48ZyB0cmFuc2Zvcm09Im1hdHJpeCguODQgMCAwIC44NCA2NSA1KSI+PHBhdGggZD0iTTkuNSAyMi45bDQuOCA2LjRhMy4xMiAzLjEyIDAgMCAxLTMgMi4ybC00LjgtNi40Yy4zLTEuNCAxLjYtMi40IDMtMi4yeiIgZmlsbD0iI2QwY2ZjZSIvPjxwYXRoIGZpbGw9IiMwMTAxMDEiIGQ9Ik00MS43IDM4LjVsNS4xLTYuNSIvPjxwYXRoIGQ9Ik00Mi45IDI3LjhMMTguNCA1OC4xIDI0IDYybDIxLjgtMjcuMyAyLjMtMi44eiIgY2xhc3M9IkUiLz48cGF0aCBmaWxsPSIjMDEwMTAxIiBkPSJNNDMuNCAyOS4zbC00LjcgNS44Ii8+PHBhdGggZD0iTTQ2LjggMzJjMy4yIDIuNiA4LjcgMS4yIDEyLjEtMy4yczMuNi05LjkuMy0xMi41bC01LjEgNi41LTIuOC0uMS0uNy0yLjcgNS4xLTYuNWMtMy4yLTIuNi04LjctMS4yLTEyLjEgMy4ycy0zLjYgOS45LS4zIDEyLjUiIGNsYXNzPSJFIi8+PHBhdGggZmlsbD0iI2E1NzkzOSIgZD0iTTI3LjMgMjZsMTEuOCAxNS43IDMuNCAyLjQgOS4xIDE0LjQtMy4yIDIuMy0xIC43LTEwLjItMTMuNi0xLjMtMy45LTExLjgtMTUuN3oiLz48cGF0aCBkPSJNMTIgMTkuOWw1LjkgNy45IDEwLjItNy42LTMuNC00LjVzNi44LTUuMSAxMC43LTQuNWMwIDAtNi42LTMtMTMuMyAxLjFTMTIgMTkuOSAxMiAxOS45eiIgY2xhc3M9IkUiLz48ZyBmaWxsPSJub25lIiBzdHJva2U9IiMwMDAiIGNsYXNzPSJCIEMgRCI+PHBhdGggZD0iTTUyIDU4LjlMNDAuOSA0My4ybC0zLjEtMi4zLTEwLjYtMTQuNy0yLjkgMi4yIDEwLjYgMTQuNyAxLjEgMy42IDExLjUgMTUuNXpNMTIuNSAxOS44bDUuOCA4IDEwLjMtNy40LTMuMy00LjZzNi45LTUgMTAuOC00LjNjMCAwLTYuNi0zLjEtMTMuMy45cy0xMC4zIDcuNC0xMC4zIDcuNHptLTIuNiAyLjlsNC43IDYuNWMtLjUgMS4zLTEuNyAyLjEtMyAyLjJsLTQuNy02LjVjLjMtMS40IDEuNi0yLjQgMy0yLjJ6Ii8+PHBhdGggZD0iTTQxLjMgMzguNWw1LjEtNi41bS0zLjUtMi43bC00LjYgNS44bTguMS0zLjFjMy4yIDIuNiA4LjcgMS4yIDEyLjEtMy4yczMuNi05LjkuMy0xMi41bC01LjEgNi41LTIuOC0uMS0uOC0yLjcgNS4xLTYuNWMtMy4yLTIuNi04LjctMS4yLTEyLjEgMy4yLTMuNCA0LjMtMy42IDkuOS0uMyAxMi41IiBjbGFzcz0iRiIvPjxwYXRoIGQ9Ik0zMC44IDQ0LjRMMTkgNTguOWw0IDMgMTAtMTIuNyIgY2xhc3M9IkYiLz48L2c+PC9nPjwvc3ZnPg==",
+                    "name" => "Home Address - 0x0000000000C1A6066c6c8B9d63e9B6E8865dC117"
+                  }
+                }}
+    end
+  end
 end

apps/indexer/lib/indexer/fetcher/token_instance.ex

@@ -72,14 +72,15 @@ defmodule Indexer.Fetcher.TokenInstance do
         {:ok, _result} = Chain.upsert_token_instance(params)
 
       result ->
-        Logger.error(fn ->
+        Logger.debug(
           [
             "failed to fetch token instance metadata for #{
               inspect({to_string(token_contract_address_hash), Decimal.to_integer(token_id)})
             }: ",
             inspect(result)
-          ]
+          ],
-        end)
+          fetcher: :token_instances
+        )
 
         :ok
     end

config/config.exs

@@ -26,8 +26,7 @@ config :logger,
     # only :indexer, but all levels
     {LoggerFileBackend, :indexer},
     {LoggerFileBackend, :indexer_token_balances},
-    {LoggerFileBackend, :failed_contract_creations},
+    {LoggerFileBackend, :token_instances},
-    {LoggerFileBackend, :addresses_without_code},
     {LoggerFileBackend, :reading_token_functions}
   ]