diff --git a/CHANGELOG.md b/CHANGELOG.md index c50170834c..e9d619acea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ - [#2470](https://github.com/poanetwork/blockscout/pull/2470) - Allow Realtime Fetcher to wait for small skips ### Fixes +- [#2779](https://github.com/poanetwork/blockscout/pull/2779) - fix fetching `latin1` encoded data - [#2783](https://github.com/poanetwork/blockscout/pull/2783) - Fix stuck value and ticker on the token page - [#2781](https://github.com/poanetwork/blockscout/pull/2781) - optimize txlist json rpc - [#2770](https://github.com/poanetwork/blockscout/pull/2770) - do not re-fetch token instances without uris diff --git a/apps/explorer/config/dev.exs b/apps/explorer/config/dev.exs index 95019470fc..a9d0f53b15 100644 --- a/apps/explorer/config/dev.exs +++ b/apps/explorer/config/dev.exs @@ -18,6 +18,11 @@ config :logger, :reading_token_functions, path: Path.absname("logs/dev/explorer/tokens/reading_functions.log"), metadata_filter: [fetcher: :token_functions] +config :logger, :token_instances, + level: :debug, + path: Path.absname("logs/dev/explorer/tokens/token_instances.log"), + metadata_filter: [fetcher: :token_instances] + import_config "dev.secret.exs" variant = diff --git a/apps/explorer/config/prod.exs b/apps/explorer/config/prod.exs index 0957ffa1f7..0137d44ae5 100644 --- a/apps/explorer/config/prod.exs +++ b/apps/explorer/config/prod.exs @@ -21,6 +21,12 @@ config :logger, :reading_token_functions, metadata_filter: [fetcher: :token_functions], rotate: %{max_bytes: 52_428_800, keep: 19} +config :logger, :token_instances, + level: :debug, + path: Path.absname("logs/prod/explorer/tokens/token_instances.log"), + metadata_filter: [fetcher: :token_instances], + rotate: %{max_bytes: 52_428_800, keep: 19} + variant = if is_nil(System.get_env("ETHEREUM_JSONRPC_VARIANT")) do "parity" diff --git a/apps/explorer/lib/explorer/token/instance_metadata_retriever.ex b/apps/explorer/lib/explorer/token/instance_metadata_retriever.ex index 6039acd038..3b06fae520 100644 --- a/apps/explorer/lib/explorer/token/instance_metadata_retriever.ex +++ b/apps/explorer/lib/explorer/token/instance_metadata_retriever.ex @@ -48,33 +48,50 @@ defmodule Explorer.Token.InstanceMetadataRetriever do Reader.query_contract(contract_address_hash, @abi, contract_functions) end - defp fetch_json(%{"tokenURI" => {:ok, [""]}}) do + def fetch_json(%{"tokenURI" => {:ok, [""]}}) do {:ok, %{error: @no_uri_error}} end - defp fetch_json(%{"tokenURI" => {:error, "(-32015) VM execution error."}}) do + def fetch_json(%{"tokenURI" => {:error, "(-32015) VM execution error."}}) do {:ok, %{error: @no_uri_error}} end - defp fetch_json(%{"tokenURI" => {:ok, ["http://" <> _ = token_uri]}}) do + def fetch_json(%{"tokenURI" => {:ok, ["http://" <> _ = token_uri]}}) do fetch_metadata(token_uri) end - defp fetch_json(%{"tokenURI" => {:ok, ["https://" <> _ = token_uri]}}) do + def fetch_json(%{"tokenURI" => {:ok, ["https://" <> _ = token_uri]}}) do fetch_metadata(token_uri) end - defp fetch_json(%{"tokenURI" => {:ok, [json]}}) do - Jason.decode(json) + def fetch_json(%{"tokenURI" => {:ok, ["data:application/json," <> json]}}) do + decoded_json = URI.decode(json) + + fetch_json(%{"tokenURI" => {:ok, [decoded_json]}}) rescue e -> - Logger.error(fn -> ["Unknown metadata format #{inspect(json)}. error #{inspect(e)}"] end) + Logger.debug(["Unknown metadata format #{inspect(json)}. error #{inspect(e)}"], + fetcher: :token_instances + ) {:error, json} end - defp fetch_json(result) do - Logger.error(fn -> ["Unknown metadata format #{inspect(result)}."] end) + def fetch_json(%{"tokenURI" => {:ok, [json]}}) do + {:ok, json} = decode_json(json) + + {:ok, %{metadata: json}} + rescue + e -> + Logger.debug(["Unknown metadata format #{inspect(json)}. error #{inspect(e)}"], + fetcher: :token_instances + ) + + {:error, json} + end + + def fetch_json(result) do + Logger.debug(["Unknown metadata format #{inspect(result)}."], fetcher: :token_instances) {:error, result} end @@ -82,7 +99,7 @@ defmodule Explorer.Token.InstanceMetadataRetriever do defp fetch_metadata(token_uri) do case HTTPoison.get(token_uri) do {:ok, %Response{body: body, status_code: 200}} -> - {:ok, json} = Jason.decode(body) + {:ok, json} = decode_json(body) {:ok, %{metadata: json}} @@ -94,8 +111,20 @@ defmodule Explorer.Token.InstanceMetadataRetriever do end rescue e -> - Logger.error(fn -> ["Could not send request to token uri #{inspect(token_uri)}. error #{inspect(e)}"] end) + Logger.debug(["Could not send request to token uri #{inspect(token_uri)}. error #{inspect(e)}"], + fetcher: :token_instances + ) {:error, :request_error} end + + defp decode_json(body) do + if String.valid?(body) do + Jason.decode(body) + else + body + |> :unicode.characters_to_binary(:latin1) + |> Jason.decode() + end + end end diff --git a/apps/explorer/test/explorer/token/instance_metadata_retriever_test.exs b/apps/explorer/test/explorer/token/instance_metadata_retriever_test.exs index 459bb53b02..4374b6c3e0 100644 --- a/apps/explorer/test/explorer/token/instance_metadata_retriever_test.exs +++ b/apps/explorer/test/explorer/token/instance_metadata_retriever_test.exs @@ -2,6 +2,7 @@ defmodule Explorer.Token.InstanceMetadataRetrieverTest do use EthereumJSONRPC.Case alias Explorer.Token.InstanceMetadataRetriever + alias Plug.Conn import Mox @@ -50,4 +51,51 @@ defmodule Explorer.Token.InstanceMetadataRetrieverTest do }) end end + + describe "fetch_json/1" do + setup do + bypass = Bypass.open() + + {:ok, bypass: bypass} + end + + test "fetches json with latin1 encoding", %{bypass: bypass} do + json = """ + { + "name": "Sérgio Mendonça" + } + """ + + Bypass.expect(bypass, "GET", "/api/card/55265", fn conn -> + Conn.resp(conn, 200, json) + end) + + assert {:ok, %{metadata: %{"name" => "Sérgio Mendonça"}}} == + InstanceMetadataRetriever.fetch_json(%{ + "tokenURI" => {:ok, ["http://localhost:#{bypass.port}/api/card/55265"]} + }) + end + + test "decodes json file in tokenURI" do + data = %{ + "tokenURI" => + {:ok, + [ + "data:application/json,{\"name\":\"Home%20Address%20-%200x0000000000C1A6066c6c8B9d63e9B6E8865dC117\",\"description\":\"This%20NFT%20can%20be%20redeemed%20on%20HomeWork%20to%20grant%20a%20controller%20the%20exclusive%20right%20to%20deploy%20contracts%20with%20arbitrary%20bytecode%20to%20the%20designated%20home%20address.\",\"image\":\"data:image/svg+xml;charset=utf-8;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAxNDQgNzIiPjxzdHlsZT48IVtDREFUQVsuQntzdHJva2UtbGluZWpvaW46cm91bmR9LkN7c3Ryb2tlLW1pdGVybGltaXQ6MTB9LkR7c3Ryb2tlLXdpZHRoOjJ9LkV7ZmlsbDojOWI5YjlhfS5Ge3N0cm9rZS1saW5lY2FwOnJvdW5kfV1dPjwvc3R5bGU+PGcgdHJhbnNmb3JtPSJtYXRyaXgoMS4wMiAwIDAgMS4wMiA4LjEgMCkiPjxwYXRoIGZpbGw9IiNmZmYiIGQ9Ik0xOSAzMmgzNHYyNEgxOXoiLz48ZyBzdHJva2U9IiMwMDAiIGNsYXNzPSJCIEMgRCI+PHBhdGggZmlsbD0iI2E1NzkzOSIgZD0iTTI1IDQwaDl2MTZoLTl6Ii8+PHBhdGggZmlsbD0iIzkyZDNmNSIgZD0iTTQwIDQwaDh2N2gtOHoiLz48cGF0aCBmaWxsPSIjZWE1YTQ3IiBkPSJNNTMgMzJIMTl2LTFsMTYtMTYgMTggMTZ6Ii8+PHBhdGggZmlsbD0ibm9uZSIgZD0iTTE5IDMyaDM0djI0SDE5eiIvPjxwYXRoIGZpbGw9IiNlYTVhNDciIGQ9Ik0yOSAyMWwtNSA1di05aDV6Ii8+PC9nPjwvZz48ZyB0cmFuc2Zvcm09Im1hdHJpeCguODQgMCAwIC44NCA2NSA1KSI+PHBhdGggZD0iTTkuNSAyMi45bDQuOCA2LjRhMy4xMiAzLjEyIDAgMCAxLTMgMi4ybC00LjgtNi40Yy4zLTEuNCAxLjYtMi40IDMtMi4yeiIgZmlsbD0iI2QwY2ZjZSIvPjxwYXRoIGZpbGw9IiMwMTAxMDEiIGQ9Ik00MS43IDM4LjVsNS4xLTYuNSIvPjxwYXRoIGQ9Ik00Mi45IDI3LjhMMTguNCA1OC4xIDI0IDYybDIxLjgtMjcuMyAyLjMtMi44eiIgY2xhc3M9IkUiLz48cGF0aCBmaWxsPSIjMDEwMTAxIiBkPSJNNDMuNCAyOS4zbC00LjcgNS44Ii8+PHBhdGggZD0iTTQ2LjggMzJjMy4yIDIuNiA4LjcgMS4yIDEyLjEtMy4yczMuNi05LjkuMy0xMi41bC01LjEgNi41LTIuOC0uMS0uNy0yLjcgNS4xLTYuNWMtMy4yLTIuNi04LjctMS4yLTEyLjEgMy4ycy0zLjYgOS45LS4zIDEyLjUiIGNsYXNzPSJFIi8+PHBhdGggZmlsbD0iI2E1NzkzOSIgZD0iTTI3LjMgMjZsMTEuOCAxNS43IDMuNCAyLjQgOS4xIDE0LjQtMy4yIDIuMy0xIC43LTEwLjItMTMuNi0xLjMtMy45LTExLjgtMTUuN3oiLz48cGF0aCBkPSJNMTIgMTkuOWw1LjkgNy45IDEwLjItNy42LTMuNC00LjVzNi44LTUuMSAxMC43LTQuNWMwIDAtNi42LTMtMTMuMyAxLjFTMTIgMTkuOSAxMiAxOS45eiIgY2xhc3M9IkUiLz48ZyBmaWxsPSJub25lIiBzdHJva2U9IiMwMDAiIGNsYXNzPSJCIEMgRCI+PHBhdGggZD0iTTUyIDU4LjlMNDAuOSA0My4ybC0zLjEtMi4zLTEwLjYtMTQuNy0yLjkgMi4yIDEwLjYgMTQuNyAxLjEgMy42IDExLjUgMTUuNXpNMTIuNSAxOS44bDUuOCA4IDEwLjMtNy40LTMuMy00LjZzNi45LTUgMTAuOC00LjNjMCAwLTYuNi0zLjEtMTMuMy45cy0xMC4zIDcuNC0xMC4zIDcuNHptLTIuNiAyLjlsNC43IDYuNWMtLjUgMS4zLTEuNyAyLjEtMyAyLjJsLTQuNy02LjVjLjMtMS40IDEuNi0yLjQgMy0yLjJ6Ii8+PHBhdGggZD0iTTQxLjMgMzguNWw1LjEtNi41bS0zLjUtMi43bC00LjYgNS44bTguMS0zLjFjMy4yIDIuNiA4LjcgMS4yIDEyLjEtMy4yczMuNi05LjkuMy0xMi41bC01LjEgNi41LTIuOC0uMS0uOC0yLjcgNS4xLTYuNWMtMy4yLTIuNi04LjctMS4yLTEyLjEgMy4yLTMuNCA0LjMtMy42IDkuOS0uMyAxMi41IiBjbGFzcz0iRiIvPjxwYXRoIGQ9Ik0zMC44IDQ0LjRMMTkgNTguOWw0IDMgMTAtMTIuNyIgY2xhc3M9IkYiLz48L2c+PC9nPjwvc3ZnPg==\"}" + ]} + } + + assert InstanceMetadataRetriever.fetch_json(data) == + {:ok, + %{ + metadata: %{ + "description" => + "This NFT can be redeemed on HomeWork to grant a controller the exclusive right to deploy contracts with arbitrary bytecode to the designated home address.", + "image" => + "data:image/svg+xml;charset=utf-8;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAxNDQgNzIiPjxzdHlsZT48IVtDREFUQVsuQntzdHJva2UtbGluZWpvaW46cm91bmR9LkN7c3Ryb2tlLW1pdGVybGltaXQ6MTB9LkR7c3Ryb2tlLXdpZHRoOjJ9LkV7ZmlsbDojOWI5YjlhfS5Ge3N0cm9rZS1saW5lY2FwOnJvdW5kfV1dPjwvc3R5bGU+PGcgdHJhbnNmb3JtPSJtYXRyaXgoMS4wMiAwIDAgMS4wMiA4LjEgMCkiPjxwYXRoIGZpbGw9IiNmZmYiIGQ9Ik0xOSAzMmgzNHYyNEgxOXoiLz48ZyBzdHJva2U9IiMwMDAiIGNsYXNzPSJCIEMgRCI+PHBhdGggZmlsbD0iI2E1NzkzOSIgZD0iTTI1IDQwaDl2MTZoLTl6Ii8+PHBhdGggZmlsbD0iIzkyZDNmNSIgZD0iTTQwIDQwaDh2N2gtOHoiLz48cGF0aCBmaWxsPSIjZWE1YTQ3IiBkPSJNNTMgMzJIMTl2LTFsMTYtMTYgMTggMTZ6Ii8+PHBhdGggZmlsbD0ibm9uZSIgZD0iTTE5IDMyaDM0djI0SDE5eiIvPjxwYXRoIGZpbGw9IiNlYTVhNDciIGQ9Ik0yOSAyMWwtNSA1di05aDV6Ii8+PC9nPjwvZz48ZyB0cmFuc2Zvcm09Im1hdHJpeCguODQgMCAwIC44NCA2NSA1KSI+PHBhdGggZD0iTTkuNSAyMi45bDQuOCA2LjRhMy4xMiAzLjEyIDAgMCAxLTMgMi4ybC00LjgtNi40Yy4zLTEuNCAxLjYtMi40IDMtMi4yeiIgZmlsbD0iI2QwY2ZjZSIvPjxwYXRoIGZpbGw9IiMwMTAxMDEiIGQ9Ik00MS43IDM4LjVsNS4xLTYuNSIvPjxwYXRoIGQ9Ik00Mi45IDI3LjhMMTguNCA1OC4xIDI0IDYybDIxLjgtMjcuMyAyLjMtMi44eiIgY2xhc3M9IkUiLz48cGF0aCBmaWxsPSIjMDEwMTAxIiBkPSJNNDMuNCAyOS4zbC00LjcgNS44Ii8+PHBhdGggZD0iTTQ2LjggMzJjMy4yIDIuNiA4LjcgMS4yIDEyLjEtMy4yczMuNi05LjkuMy0xMi41bC01LjEgNi41LTIuOC0uMS0uNy0yLjcgNS4xLTYuNWMtMy4yLTIuNi04LjctMS4yLTEyLjEgMy4ycy0zLjYgOS45LS4zIDEyLjUiIGNsYXNzPSJFIi8+PHBhdGggZmlsbD0iI2E1NzkzOSIgZD0iTTI3LjMgMjZsMTEuOCAxNS43IDMuNCAyLjQgOS4xIDE0LjQtMy4yIDIuMy0xIC43LTEwLjItMTMuNi0xLjMtMy45LTExLjgtMTUuN3oiLz48cGF0aCBkPSJNMTIgMTkuOWw1LjkgNy45IDEwLjItNy42LTMuNC00LjVzNi44LTUuMSAxMC43LTQuNWMwIDAtNi42LTMtMTMuMyAxLjFTMTIgMTkuOSAxMiAxOS45eiIgY2xhc3M9IkUiLz48ZyBmaWxsPSJub25lIiBzdHJva2U9IiMwMDAiIGNsYXNzPSJCIEMgRCI+PHBhdGggZD0iTTUyIDU4LjlMNDAuOSA0My4ybC0zLjEtMi4zLTEwLjYtMTQuNy0yLjkgMi4yIDEwLjYgMTQuNyAxLjEgMy42IDExLjUgMTUuNXpNMTIuNSAxOS44bDUuOCA4IDEwLjMtNy40LTMuMy00LjZzNi45LTUgMTAuOC00LjNjMCAwLTYuNi0zLjEtMTMuMy45cy0xMC4zIDcuNC0xMC4zIDcuNHptLTIuNiAyLjlsNC43IDYuNWMtLjUgMS4zLTEuNyAyLjEtMyAyLjJsLTQuNy02LjVjLjMtMS40IDEuNi0yLjQgMy0yLjJ6Ii8+PHBhdGggZD0iTTQxLjMgMzguNWw1LjEtNi41bS0zLjUtMi43bC00LjYgNS44bTguMS0zLjFjMy4yIDIuNiA4LjcgMS4yIDEyLjEtMy4yczMuNi05LjkuMy0xMi41bC01LjEgNi41LTIuOC0uMS0uOC0yLjcgNS4xLTYuNWMtMy4yLTIuNi04LjctMS4yLTEyLjEgMy4yLTMuNCA0LjMtMy42IDkuOS0uMyAxMi41IiBjbGFzcz0iRiIvPjxwYXRoIGQ9Ik0zMC44IDQ0LjRMMTkgNTguOWw0IDMgMTAtMTIuNyIgY2xhc3M9IkYiLz48L2c+PC9nPjwvc3ZnPg==", + "name" => "Home Address - 0x0000000000C1A6066c6c8B9d63e9B6E8865dC117" + } + }} + end + end end diff --git a/apps/indexer/lib/indexer/fetcher/token_instance.ex b/apps/indexer/lib/indexer/fetcher/token_instance.ex index 14936d05af..058dee9e2d 100644 --- a/apps/indexer/lib/indexer/fetcher/token_instance.ex +++ b/apps/indexer/lib/indexer/fetcher/token_instance.ex @@ -72,14 +72,15 @@ defmodule Indexer.Fetcher.TokenInstance do {:ok, _result} = Chain.upsert_token_instance(params) result -> - Logger.error(fn -> + Logger.debug( [ "failed to fetch token instance metadata for #{ inspect({to_string(token_contract_address_hash), Decimal.to_integer(token_id)}) }: ", inspect(result) - ] - end) + ], + fetcher: :token_instances + ) :ok end diff --git a/config/config.exs b/config/config.exs index 17d60558e6..c6a4ed5043 100644 --- a/config/config.exs +++ b/config/config.exs @@ -26,8 +26,7 @@ config :logger, # only :indexer, but all levels {LoggerFileBackend, :indexer}, {LoggerFileBackend, :indexer_token_balances}, - {LoggerFileBackend, :failed_contract_creations}, - {LoggerFileBackend, :addresses_without_code}, + {LoggerFileBackend, :token_instances}, {LoggerFileBackend, :reading_token_functions} ]