From ba481afc8db15beab85bade0c0c004c6f1b49ebf Mon Sep 17 00:00:00 2001 From: Nikita Pozdniakov Date: Mon, 28 Aug 2023 11:08:37 +0300 Subject: [PATCH] Add batches to TokenInstance fetchers --- CHANGELOG.md | 1 + .../lib/ethereum_jsonrpc/encoder.ex | 4 +- apps/explorer/lib/explorer/chain.ex | 11 + .../instance_metadata_retriever_test.exs | 249 +++--------------- .../indexer/fetcher/token_instance/helper.ex | 174 ++++++++++-- .../token_instance/metadata_retriever.ex | 102 ++----- .../fetcher/token_instance/realtime.ex | 12 +- .../indexer/fetcher/token_instance/retry.ex | 18 +- .../fetcher/token_instance/sanitize.ex | 14 +- .../fetcher/token_instance/helper_test.exs | 178 +++++++++++++ apps/indexer/test/test_helper.exs | 1 + config/runtime.exs | 7 +- docker-compose/envs/common-blockscout.env | 3 + docker/Makefile | 9 + 14 files changed, 435 insertions(+), 348 deletions(-) create mode 100644 apps/indexer/test/indexer/fetcher/token_instance/helper_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index 5435dba7cc..5dfacc16cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Features +- [#8313](https://github.com/blockscout/blockscout/pull/8313) - Add batches to TokenInstance fetchers - [#8181](https://github.com/blockscout/blockscout/pull/8181) - Insert current token balances placeholders along with historical - [#8210](https://github.com/blockscout/blockscout/pull/8210) - Drop address foreign keys - [#8292](https://github.com/blockscout/blockscout/pull/8292) - Add ETHEREUM_JSONRPC_WAIT_PER_TIMEOUT env var diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/encoder.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/encoder.ex index 221c082b02..37573679aa 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/encoder.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/encoder.ex @@ -69,7 +69,7 @@ defmodule EthereumJSONRPC.Encoder do end end - def decode_result(result, selectors, _leave_error_as_map) when is_list(selectors) do + def decode_result(%{id: id, result: _result} = result, selectors, _leave_error_as_map) when is_list(selectors) do selectors |> Enum.map(fn selector -> try do @@ -78,7 +78,7 @@ defmodule EthereumJSONRPC.Encoder do _ -> :error end end) - |> Enum.find(fn decode -> + |> Enum.find({id, {:error, :unable_to_decode}}, fn decode -> case decode do {_id, {:ok, _}} -> true _ -> false diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index 264ce9d67b..2c9ffe38af 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -4690,6 +4690,9 @@ defmodule Explorer.Chain do end end + @doc """ + Expects map of change params. Inserts using on_conflict: :replace_all + """ @spec upsert_token_instance(map()) :: {:ok, Instance.t()} | {:error, Ecto.Changeset.t()} def upsert_token_instance(params) do changeset = Instance.changeset(%Instance{}, params) @@ -4700,6 +4703,14 @@ defmodule Explorer.Chain do ) end + @doc """ + Inserts list of token instances via upsert_token_instance/1. + """ + @spec upsert_token_instances_list([map()]) :: list() + def upsert_token_instances_list(instances) do + Enum.map(instances, &upsert_token_instance/1) + end + @doc """ Update a new `t:Token.t/0` record. diff --git a/apps/explorer/test/explorer/token/instance_metadata_retriever_test.exs b/apps/explorer/test/explorer/token/instance_metadata_retriever_test.exs index 0e5ba979a9..a9255d04c7 100644 --- a/apps/explorer/test/explorer/token/instance_metadata_retriever_test.exs +++ b/apps/explorer/test/explorer/token/instance_metadata_retriever_test.exs @@ -1,7 +1,6 @@ defmodule Explorer.Token.MetadataRetrieverTest do use EthereumJSONRPC.Case - alias EthereumJSONRPC.Encoder alias Indexer.Fetcher.TokenInstance.MetadataRetriever alias Plug.Conn @@ -164,28 +163,7 @@ defmodule Explorer.Token.MetadataRetrieverTest do end) assert {:ok, %{metadata: %{"name" => "Sérgio Mendonça"}}} == - MetadataRetriever.fetch_json(%{ - "c87b56dd" => {:ok, ["http://localhost:#{bypass.port}#{path}"]} - }) - end - - test "fetches json metadata for kitties" do - Application.put_env(:explorer, :http_adapter, Explorer.Mox.HTTPoison) - - result = - "{\"id\":100500,\"name\":\"KittyBlue_2_Lemonade\",\"generation\":20,\"genes\":\"623509754227292470437941473598751240781530569131665917719736997423495595\",\"created_at\":\"2017-12-06T01:56:27.000Z\",\"birthday\":\"2017-12-06T00:00:00.000Z\",\"image_url\":\"https://img.cryptokitties.co/0x06012c8cf97bead5deae237070f9587f8e7a266d/100500.svg\",\"image_url_cdn\":\"https://img.cn.cryptokitties.co/0x06012c8cf97bead5deae237070f9587f8e7a266d/100500.svg\",\"color\":\"strawberry\",\"background_color\":\"#ffe0e5\",\"bio\":\"Shalom! I'm KittyBlue_2_Lemonade. I'm a professional Foreign Film Director and I love cantaloupe. I'm convinced that the world is flat. One day I'll prove it. It's pawesome to meet you!\",\"kitty_type\":null,\"is_fancy\":false,\"is_exclusive\":false,\"is_special_edition\":false,\"fancy_type\":null,\"language\":\"en\",\"is_prestige\":false,\"prestige_type\":null,\"prestige_ranking\":null,\"prestige_time_limit\":null,\"status\":{\"is_ready\":true,\"is_gestating\":false,\"cooldown\":1410310201506,\"dynamic_cooldown\":1475064986478,\"cooldown_index\":10,\"cooldown_end_block\":0,\"pending_tx_type\":null,\"pending_tx_since\":null},\"purrs\":{\"count\":1,\"is_purred\":false},\"watchlist\":{\"count\":0,\"is_watchlisted\":false},\"hatcher\":{\"address\":\"0x7b9ea9ac69b8fde875554321472c732eeff06ca0\",\"image\":\"14\",\"nickname\":\"KittyBlu\",\"hasDapper\":false,\"twitter_id\":null,\"twitter_image_url\":null,\"twitter_handle\":null},\"auction\":{},\"offer\":{},\"owner\":{\"address\":\"0x7b9ea9ac69b8fde875554321472c732eeff06ca0\",\"hasDapper\":false,\"twitter_id\":null,\"twitter_image_url\":null,\"twitter_handle\":null,\"image\":\"14\",\"nickname\":\"KittyBlu\"},\"matron\":{\"id\":46234,\"name\":\"KittyBlue_1_Limegreen\",\"generation\":10,\"enhanced_cattributes\":[{\"type\":\"body\",\"kittyId\":19631,\"position\":105,\"description\":\"cymric\"},{\"type\":\"coloreyes\",\"kittyId\":40356,\"position\":263,\"description\":\"limegreen\"},{\"type\":\"eyes\",\"kittyId\":3185,\"position\":16,\"description\":\"raisedbrow\"},{\"type\":\"pattern\",\"kittyId\":46234,\"position\":-1,\"description\":\"totesbasic\"},{\"type\":\"mouth\",\"kittyId\":46234,\"position\":-1,\"description\":\"happygokitty\"},{\"type\":\"colorprimary\",\"kittyId\":46234,\"position\":-1,\"description\":\"greymatter\"},{\"type\":\"colorsecondary\",\"kittyId\":46234,\"position\":-1,\"description\":\"lemonade\"},{\"type\":\"colortertiary\",\"kittyId\":46234,\"position\":-1,\"description\":\"granitegrey\"}],\"owner_wallet_address\":\"0x7b9ea9ac69b8fde875554321472c732eeff06ca0\",\"owner\":{\"address\":\"0x7b9ea9ac69b8fde875554321472c732eeff06ca0\"},\"created_at\":\"2017-12-03T21:29:17.000Z\",\"image_url\":\"https://img.cryptokitties.co/0x06012c8cf97bead5deae237070f9587f8e7a266d/46234.svg\",\"image_url_cdn\":\"https://img.cn.cryptokitties.co/0x06012c8cf97bead5deae237070f9587f8e7a266d/46234.svg\",\"color\":\"limegreen\",\"is_fancy\":false,\"kitty_type\":null,\"is_exclusive\":false,\"is_special_edition\":false,\"fancy_type\":null,\"status\":{\"is_ready\":true,\"is_gestating\":false,\"cooldown\":1486487069384},\"hatched\":true,\"wrapped\":false,\"image_url_png\":\"https://img.cryptokitties.co/0x06012c8cf97bead5deae237070f9587f8e7a266d/46234.png\"},\"sire\":{\"id\":82090,\"name\":null,\"generation\":19,\"enhanced_cattributes\":[{\"type\":\"body\",\"kittyId\":82090,\"position\":-1,\"description\":\"himalayan\"},{\"type\":\"coloreyes\",\"kittyId\":82090,\"position\":-1,\"description\":\"strawberry\"},{\"type\":\"eyes\",\"kittyId\":82090,\"position\":-1,\"description\":\"thicccbrowz\"},{\"type\":\"pattern\",\"kittyId\":82090,\"position\":-1,\"description\":\"totesbasic\"},{\"type\":\"mouth\",\"kittyId\":82090,\"position\":-1,\"description\":\"pouty\"},{\"type\":\"colorprimary\",\"kittyId\":82090,\"position\":-1,\"description\":\"aquamarine\"},{\"type\":\"colorsecondary\",\"kittyId\":82090,\"position\":-1,\"description\":\"chocolate\"},{\"type\":\"colortertiary\",\"kittyId\":82090,\"position\":-1,\"description\":\"granitegrey\"}],\"owner_wallet_address\":\"0x798fdad0cedc4b298fc7d53a982fa0c5f447eaa5\",\"owner\":{\"address\":\"0x798fdad0cedc4b298fc7d53a982fa0c5f447eaa5\"},\"created_at\":\"2017-12-05T06:30:05.000Z\",\"image_url\":\"https://img.cryptokitties.co/0x06012c8cf97bead5deae237070f9587f8e7a266d/82090.svg\",\"image_url_cdn\":\"https://img.cn.cryptokitties.co/0x06012c8cf97bead5deae237070f9587f8e7a266d/82090.svg\",\"color\":\"strawberry\",\"is_fancy\":false,\"is_exclusive\":false,\"is_special_edition\":false,\"fancy_type\":null,\"status\":{\"is_ready\":true,\"is_gestating\":false,\"cooldown\":1486619010030},\"kitty_type\":null,\"hatched\":true,\"wrapped\":false,\"image_url_png\":\"https://img.cryptokitties.co/0x06012c8cf97bead5deae237070f9587f8e7a266d/82090.png\"},\"children\":[],\"hatched\":true,\"wrapped\":false,\"enhanced_cattributes\":[{\"type\":\"colorprimary\",\"description\":\"greymatter\",\"position\":null,\"kittyId\":100500},{\"type\":\"coloreyes\",\"description\":\"strawberry\",\"position\":null,\"kittyId\":100500},{\"type\":\"body\",\"description\":\"himalayan\",\"position\":null,\"kittyId\":100500},{\"type\":\"colorsecondary\",\"description\":\"lemonade\",\"position\":null,\"kittyId\":100500},{\"type\":\"mouth\",\"description\":\"pouty\",\"position\":null,\"kittyId\":100500},{\"type\":\"pattern\",\"description\":\"totesbasic\",\"position\":null,\"kittyId\":100500},{\"type\":\"eyes\",\"description\":\"thicccbrowz\",\"position\":null,\"kittyId\":100500},{\"type\":\"colortertiary\",\"description\":\"kittencream\",\"position\":null,\"kittyId\":100500},{\"type\":\"secret\",\"description\":\"se5\",\"position\":-1,\"kittyId\":100500},{\"type\":\"purrstige\",\"description\":\"pu20\",\"position\":-1,\"kittyId\":100500}],\"variation\":null,\"variation_ranking\":null,\"image_url_png\":\"https://img.cryptokitties.co/0x06012c8cf97bead5deae237070f9587f8e7a266d/100500.png\",\"items\":[]}" - - Explorer.Mox.HTTPoison - |> expect(:get, fn "https://api.cryptokitties.co/kitties/100500", _headers, _options -> - {:ok, %HTTPoison.Response{status_code: 200, body: result}} - end) - - {:ok, %{metadata: metadata}} = - MetadataRetriever.fetch_metadata("0x06012c8cf97bead5deae237070f9587f8e7a266d", 100_500) - - assert Map.get(metadata, "name") == "KittyBlue_2_Lemonade" - - Application.put_env(:explorer, :http_adapter, HTTPoison) + MetadataRetriever.fetch_json({:ok, ["http://localhost:#{bypass.port}#{path}"]}) end test "fetches json metadata when HTTP status 301", %{bypass: bypass} do @@ -217,112 +195,12 @@ defmodule Explorer.Token.MetadataRetrieverTest do assert Map.get(metadata, "attributes") == Jason.decode!(attributes) end - test "replace {id} with actual token_id", %{bypass: bypass} do - json = """ - { - "name": "Sérgio Mendonça {id}" - } - """ - - abi = - [ - %{ - "type" => "function", - "stateMutability" => "nonpayable", - "payable" => false, - "outputs" => [], - "name" => "tokenURI", - "inputs" => [ - %{"type" => "string", "name" => "name", "internalType" => "string"} - ] - } - ] - |> ABI.parse_specification() - |> Enum.at(0) - - encoded_url = - abi - |> Encoder.encode_function_call(["http://localhost:#{bypass.port}/api/card/{id}"]) - |> String.replace("4cf12d26", "") - - EthereumJSONRPC.Mox - |> expect(:json_rpc, fn [ - %{ - id: 0, - jsonrpc: "2.0", - method: "eth_call", - params: [ - %{ - data: - "0xc87b56dd0000000000000000000000000000000000000000000000000000000000000309", - to: "0x5caebd3b32e210e85ce3e9d51638b9c445481567" - }, - "latest" - ] - } - ], - _options -> - {:ok, - [ - %{ - id: 0, - jsonrpc: "2.0", - error: %{code: -32000, message: "execution reverted"} - } - ]} - end) - |> expect(:json_rpc, fn [ - %{ - id: 0, - jsonrpc: "2.0", - method: "eth_call", - params: [ - %{ - data: - "0x0e89341c0000000000000000000000000000000000000000000000000000000000000309", - to: "0x5caebd3b32e210e85ce3e9d51638b9c445481567" - }, - "latest" - ] - } - ], - _options -> + test "decodes json file in tokenURI" do + data = {:ok, [ - %{ - id: 0, - jsonrpc: "2.0", - result: encoded_url - } + "data:application/json,{\"name\":\"Home%20Address%20-%200x0000000000C1A6066c6c8B9d63e9B6E8865dC117\",\"description\":\"This%20NFT%20can%20be%20redeemed%20on%20HomeWork%20to%20grant%20a%20controller%20the%20exclusive%20right%20to%20deploy%20contracts%20with%20arbitrary%20bytecode%20to%20the%20designated%20home%20address.\",\"image\":\"data:image/svg+xml;charset=utf-8;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAxNDQgNzIiPjxzdHlsZT48IVtDREFUQVsuQntzdHJva2UtbGluZWpvaW46cm91bmR9LkN7c3Ryb2tlLW1pdGVybGltaXQ6MTB9LkR7c3Ryb2tlLXdpZHRoOjJ9LkV7ZmlsbDojOWI5YjlhfS5Ge3N0cm9rZS1saW5lY2FwOnJvdW5kfV1dPjwvc3R5bGU+PGcgdHJhbnNmb3JtPSJtYXRyaXgoMS4wMiAwIDAgMS4wMiA4LjEgMCkiPjxwYXRoIGZpbGw9IiNmZmYiIGQ9Ik0xOSAzMmgzNHYyNEgxOXoiLz48ZyBzdHJva2U9IiMwMDAiIGNsYXNzPSJCIEMgRCI+PHBhdGggZmlsbD0iI2E1NzkzOSIgZD0iTTI1IDQwaDl2MTZoLTl6Ii8+PHBhdGggZmlsbD0iIzkyZDNmNSIgZD0iTTQwIDQwaDh2N2gtOHoiLz48cGF0aCBmaWxsPSIjZWE1YTQ3IiBkPSJNNTMgMzJIMTl2LTFsMTYtMTYgMTggMTZ6Ii8+PHBhdGggZmlsbD0ibm9uZSIgZD0iTTE5IDMyaDM0djI0SDE5eiIvPjxwYXRoIGZpbGw9IiNlYTVhNDciIGQ9Ik0yOSAyMWwtNSA1di05aDV6Ii8+PC9nPjwvZz48ZyB0cmFuc2Zvcm09Im1hdHJpeCguODQgMCAwIC44NCA2NSA1KSI+PHBhdGggZD0iTTkuNSAyMi45bDQuOCA2LjRhMy4xMiAzLjEyIDAgMCAxLTMgMi4ybC00LjgtNi40Yy4zLTEuNCAxLjYtMi40IDMtMi4yeiIgZmlsbD0iI2QwY2ZjZSIvPjxwYXRoIGZpbGw9IiMwMTAxMDEiIGQ9Ik00MS43IDM4LjVsNS4xLTYuNSIvPjxwYXRoIGQ9Ik00Mi45IDI3LjhMMTguNCA1OC4xIDI0IDYybDIxLjgtMjcuMyAyLjMtMi44eiIgY2xhc3M9IkUiLz48cGF0aCBmaWxsPSIjMDEwMTAxIiBkPSJNNDMuNCAyOS4zbC00LjcgNS44Ii8+PHBhdGggZD0iTTQ2LjggMzJjMy4yIDIuNiA4LjcgMS4yIDEyLjEtMy4yczMuNi05LjkuMy0xMi41bC01LjEgNi41LTIuOC0uMS0uNy0yLjcgNS4xLTYuNWMtMy4yLTIuNi04LjctMS4yLTEyLjEgMy4ycy0zLjYgOS45LS4zIDEyLjUiIGNsYXNzPSJFIi8+PHBhdGggZmlsbD0iI2E1NzkzOSIgZD0iTTI3LjMgMjZsMTEuOCAxNS43IDMuNCAyLjQgOS4xIDE0LjQtMy4yIDIuMy0xIC43LTEwLjItMTMuNi0xLjMtMy45LTExLjgtMTUuN3oiLz48cGF0aCBkPSJNMTIgMTkuOWw1LjkgNy45IDEwLjItNy42LTMuNC00LjVzNi44LTUuMSAxMC43LTQuNWMwIDAtNi42LTMtMTMuMyAxLjFTMTIgMTkuOSAxMiAxOS45eiIgY2xhc3M9IkUiLz48ZyBmaWxsPSJub25lIiBzdHJva2U9IiMwMDAiIGNsYXNzPSJCIEMgRCI+PHBhdGggZD0iTTUyIDU4LjlMNDAuOSA0My4ybC0zLjEtMi4zLTEwLjYtMTQuNy0yLjkgMi4yIDEwLjYgMTQuNyAxLjEgMy42IDExLjUgMTUuNXpNMTIuNSAxOS44bDUuOCA4IDEwLjMtNy40LTMuMy00LjZzNi45LTUgMTAuOC00LjNjMCAwLTYuNi0zLjEtMTMuMy45cy0xMC4zIDcuNC0xMC4zIDcuNHptLTIuNiAyLjlsNC43IDYuNWMtLjUgMS4zLTEuNyAyLjEtMyAyLjJsLTQuNy02LjVjLjMtMS40IDEuNi0yLjQgMy0yLjJ6Ii8+PHBhdGggZD0iTTQxLjMgMzguNWw1LjEtNi41bS0zLjUtMi43bC00LjYgNS44bTguMS0zLjFjMy4yIDIuNiA4LjcgMS4yIDEyLjEtMy4yczMuNi05LjkuMy0xMi41bC01LjEgNi41LTIuOC0uMS0uOC0yLjcgNS4xLTYuNWMtMy4yLTIuNi04LjctMS4yLTEyLjEgMy4yLTMuNCA0LjMtMy42IDkuOS0uMyAxMi41IiBjbGFzcz0iRiIvPjxwYXRoIGQ9Ik0zMC44IDQ0LjRMMTkgNTguOWw0IDMgMTAtMTIuNyIgY2xhc3M9IkYiLz48L2c+PC9nPjwvc3ZnPg==\"}" ]} - end) - - Bypass.expect( - bypass, - "GET", - "/api/card/0000000000000000000000000000000000000000000000000000000000000309", - fn conn -> - Conn.resp(conn, 200, json) - end - ) - - assert {:ok, - %{ - metadata: %{ - "name" => "Sérgio Mendonça 0000000000000000000000000000000000000000000000000000000000000309" - } - }} == - MetadataRetriever.fetch_metadata("0x5caebd3b32e210e85ce3e9d51638b9c445481567", 777) - end - - test "decodes json file in tokenURI" do - data = %{ - "c87b56dd" => - {:ok, - [ - "data:application/json,{\"name\":\"Home%20Address%20-%200x0000000000C1A6066c6c8B9d63e9B6E8865dC117\",\"description\":\"This%20NFT%20can%20be%20redeemed%20on%20HomeWork%20to%20grant%20a%20controller%20the%20exclusive%20right%20to%20deploy%20contracts%20with%20arbitrary%20bytecode%20to%20the%20designated%20home%20address.\",\"image\":\"data:image/svg+xml;charset=utf-8;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAxNDQgNzIiPjxzdHlsZT48IVtDREFUQVsuQntzdHJva2UtbGluZWpvaW46cm91bmR9LkN7c3Ryb2tlLW1pdGVybGltaXQ6MTB9LkR7c3Ryb2tlLXdpZHRoOjJ9LkV7ZmlsbDojOWI5YjlhfS5Ge3N0cm9rZS1saW5lY2FwOnJvdW5kfV1dPjwvc3R5bGU+PGcgdHJhbnNmb3JtPSJtYXRyaXgoMS4wMiAwIDAgMS4wMiA4LjEgMCkiPjxwYXRoIGZpbGw9IiNmZmYiIGQ9Ik0xOSAzMmgzNHYyNEgxOXoiLz48ZyBzdHJva2U9IiMwMDAiIGNsYXNzPSJCIEMgRCI+PHBhdGggZmlsbD0iI2E1NzkzOSIgZD0iTTI1IDQwaDl2MTZoLTl6Ii8+PHBhdGggZmlsbD0iIzkyZDNmNSIgZD0iTTQwIDQwaDh2N2gtOHoiLz48cGF0aCBmaWxsPSIjZWE1YTQ3IiBkPSJNNTMgMzJIMTl2LTFsMTYtMTYgMTggMTZ6Ii8+PHBhdGggZmlsbD0ibm9uZSIgZD0iTTE5IDMyaDM0djI0SDE5eiIvPjxwYXRoIGZpbGw9IiNlYTVhNDciIGQ9Ik0yOSAyMWwtNSA1di05aDV6Ii8+PC9nPjwvZz48ZyB0cmFuc2Zvcm09Im1hdHJpeCguODQgMCAwIC44NCA2NSA1KSI+PHBhdGggZD0iTTkuNSAyMi45bDQuOCA2LjRhMy4xMiAzLjEyIDAgMCAxLTMgMi4ybC00LjgtNi40Yy4zLTEuNCAxLjYtMi40IDMtMi4yeiIgZmlsbD0iI2QwY2ZjZSIvPjxwYXRoIGZpbGw9IiMwMTAxMDEiIGQ9Ik00MS43IDM4LjVsNS4xLTYuNSIvPjxwYXRoIGQ9Ik00Mi45IDI3LjhMMTguNCA1OC4xIDI0IDYybDIxLjgtMjcuMyAyLjMtMi44eiIgY2xhc3M9IkUiLz48cGF0aCBmaWxsPSIjMDEwMTAxIiBkPSJNNDMuNCAyOS4zbC00LjcgNS44Ii8+PHBhdGggZD0iTTQ2LjggMzJjMy4yIDIuNiA4LjcgMS4yIDEyLjEtMy4yczMuNi05LjkuMy0xMi41bC01LjEgNi41LTIuOC0uMS0uNy0yLjcgNS4xLTYuNWMtMy4yLTIuNi04LjctMS4yLTEyLjEgMy4ycy0zLjYgOS45LS4zIDEyLjUiIGNsYXNzPSJFIi8+PHBhdGggZmlsbD0iI2E1NzkzOSIgZD0iTTI3LjMgMjZsMTEuOCAxNS43IDMuNCAyLjQgOS4xIDE0LjQtMy4yIDIuMy0xIC43LTEwLjItMTMuNi0xLjMtMy45LTExLjgtMTUuN3oiLz48cGF0aCBkPSJNMTIgMTkuOWw1LjkgNy45IDEwLjItNy42LTMuNC00LjVzNi44LTUuMSAxMC43LTQuNWMwIDAtNi42LTMtMTMuMyAxLjFTMTIgMTkuOSAxMiAxOS45eiIgY2xhc3M9IkUiLz48ZyBmaWxsPSJub25lIiBzdHJva2U9IiMwMDAiIGNsYXNzPSJCIEMgRCI+PHBhdGggZD0iTTUyIDU4LjlMNDAuOSA0My4ybC0zLjEtMi4zLTEwLjYtMTQuNy0yLjkgMi4yIDEwLjYgMTQuNyAxLjEgMy42IDExLjUgMTUuNXpNMTIuNSAxOS44bDUuOCA4IDEwLjMtNy40LTMuMy00LjZzNi45LTUgMTAuOC00LjNjMCAwLTYuNi0zLjEtMTMuMy45cy0xMC4zIDcuNC0xMC4zIDcuNHptLTIuNiAyLjlsNC43IDYuNWMtLjUgMS4zLTEuNyAyLjEtMyAyLjJsLTQuNy02LjVjLjMtMS40IDEuNi0yLjQgMy0yLjJ6Ii8+PHBhdGggZD0iTTQxLjMgMzguNWw1LjEtNi41bS0zLjUtMi43bC00LjYgNS44bTguMS0zLjFjMy4yIDIuNiA4LjcgMS4yIDEyLjEtMy4yczMuNi05LjkuMy0xMi41bC01LjEgNi41LTIuOC0uMS0uOC0yLjcgNS4xLTYuNWMtMy4yLTIuNi04LjctMS4yLTEyLjEgMy4yLTMuNCA0LjMtMy42IDkuOS0uMyAxMi41IiBjbGFzcz0iRiIvPjxwYXRoIGQ9Ik0zMC44IDQ0LjRMMTkgNTguOWw0IDMgMTAtMTIuNyIgY2xhc3M9IkYiLz48L2c+PC9nPjwvc3ZnPg==\"}" - ]} - } assert MetadataRetriever.fetch_json(data) == {:ok, @@ -338,13 +216,11 @@ defmodule Explorer.Token.MetadataRetrieverTest do end test "decodes base64 encoded json file in tokenURI" do - data = %{ - "c87b56dd" => - {:ok, - [ - "data:application/json;base64,eyJuYW1lIjogIi54ZGFpIiwgImRlc2NyaXB0aW9uIjogIlB1bmsgRG9tYWlucyBkaWdpdGFsIGlkZW50aXR5LiBWaXNpdCBodHRwczovL3B1bmsuZG9tYWlucy8iLCAiaW1hZ2UiOiAiZGF0YTppbWFnZS9zdmcreG1sO2Jhc2U2NCxQSE4yWnlCNGJXeHVjejBpYUhSMGNEb3ZMM2QzZHk1M015NXZjbWN2TWpBd01DOXpkbWNpSUhacFpYZENiM2c5SWpBZ01DQTFNREFnTlRBd0lpQjNhV1IwYUQwaU5UQXdJaUJvWldsbmFIUTlJalV3TUNJK1BHUmxabk0rUEd4cGJtVmhja2R5WVdScFpXNTBJR2xrUFNKbmNtRmtJaUI0TVQwaU1DVWlJSGt4UFNJd0pTSWdlREk5SWpFd01DVWlJSGt5UFNJd0pTSStQSE4wYjNBZ2IyWm1jMlYwUFNJd0pTSWdjM1I1YkdVOUluTjBiM0F0WTI5c2IzSTZjbWRpS0RVNExERTNMREV4TmlrN2MzUnZjQzF2Y0dGamFYUjVPakVpSUM4K1BITjBiM0FnYjJabWMyVjBQU0l4TURBbElpQnpkSGxzWlQwaWMzUnZjQzFqYjJ4dmNqcHlaMklvTVRFMkxESTFMREUzS1R0emRHOXdMVzl3WVdOcGRIazZNU0lnTHo0OEwyeHBibVZoY2tkeVlXUnBaVzUwUGp3dlpHVm1jejQ4Y21WamRDQjRQU0l3SWlCNVBTSXdJaUIzYVdSMGFEMGlOVEF3SWlCb1pXbG5hSFE5SWpVd01DSWdabWxzYkQwaWRYSnNLQ05uY21Ga0tTSXZQangwWlhoMElIZzlJalV3SlNJZ2VUMGlOVEFsSWlCa2IyMXBibUZ1ZEMxaVlYTmxiR2x1WlQwaWJXbGtaR3hsSWlCbWFXeHNQU0ozYUdsMFpTSWdkR1Y0ZEMxaGJtTm9iM0k5SW0xcFpHUnNaU0lnWm05dWRDMXphWHBsUFNKNExXeGhjbWRsSWo0dWVHUmhhVHd2ZEdWNGRENDhkR1Y0ZENCNFBTSTFNQ1VpSUhrOUlqY3dKU0lnWkc5dGFXNWhiblF0WW1GelpXeHBibVU5SW0xcFpHUnNaU0lnWm1sc2JEMGlkMmhwZEdVaUlIUmxlSFF0WVc1amFHOXlQU0p0YVdSa2JHVWlQbkIxYm1zdVpHOXRZV2x1Y3p3dmRHVjRkRDQ4TDNOMlp6ND0ifQ==" - ]} - } + data = + {:ok, + [ + "data:application/json;base64,eyJuYW1lIjogIi54ZGFpIiwgImRlc2NyaXB0aW9uIjogIlB1bmsgRG9tYWlucyBkaWdpdGFsIGlkZW50aXR5LiBWaXNpdCBodHRwczovL3B1bmsuZG9tYWlucy8iLCAiaW1hZ2UiOiAiZGF0YTppbWFnZS9zdmcreG1sO2Jhc2U2NCxQSE4yWnlCNGJXeHVjejBpYUhSMGNEb3ZMM2QzZHk1M015NXZjbWN2TWpBd01DOXpkbWNpSUhacFpYZENiM2c5SWpBZ01DQTFNREFnTlRBd0lpQjNhV1IwYUQwaU5UQXdJaUJvWldsbmFIUTlJalV3TUNJK1BHUmxabk0rUEd4cGJtVmhja2R5WVdScFpXNTBJR2xrUFNKbmNtRmtJaUI0TVQwaU1DVWlJSGt4UFNJd0pTSWdlREk5SWpFd01DVWlJSGt5UFNJd0pTSStQSE4wYjNBZ2IyWm1jMlYwUFNJd0pTSWdjM1I1YkdVOUluTjBiM0F0WTI5c2IzSTZjbWRpS0RVNExERTNMREV4TmlrN2MzUnZjQzF2Y0dGamFYUjVPakVpSUM4K1BITjBiM0FnYjJabWMyVjBQU0l4TURBbElpQnpkSGxzWlQwaWMzUnZjQzFqYjJ4dmNqcHlaMklvTVRFMkxESTFMREUzS1R0emRHOXdMVzl3WVdOcGRIazZNU0lnTHo0OEwyeHBibVZoY2tkeVlXUnBaVzUwUGp3dlpHVm1jejQ4Y21WamRDQjRQU0l3SWlCNVBTSXdJaUIzYVdSMGFEMGlOVEF3SWlCb1pXbG5hSFE5SWpVd01DSWdabWxzYkQwaWRYSnNLQ05uY21Ga0tTSXZQangwWlhoMElIZzlJalV3SlNJZ2VUMGlOVEFsSWlCa2IyMXBibUZ1ZEMxaVlYTmxiR2x1WlQwaWJXbGtaR3hsSWlCbWFXeHNQU0ozYUdsMFpTSWdkR1Y0ZEMxaGJtTm9iM0k5SW0xcFpHUnNaU0lnWm05dWRDMXphWHBsUFNKNExXeGhjbWRsSWo0dWVHUmhhVHd2ZEdWNGRENDhkR1Y0ZENCNFBTSTFNQ1VpSUhrOUlqY3dKU0lnWkc5dGFXNWhiblF0WW1GelpXeHBibVU5SW0xcFpHUnNaU0lnWm1sc2JEMGlkMmhwZEdVaUlIUmxlSFF0WVc1amFHOXlQU0p0YVdSa2JHVWlQbkIxYm1zdVpHOXRZV2x1Y3p3dmRHVjRkRDQ4TDNOMlp6ND0ifQ==" + ]} assert MetadataRetriever.fetch_json(data) == {:ok, @@ -359,13 +235,11 @@ defmodule Explorer.Token.MetadataRetrieverTest do end test "decodes base64 encoded json file (with unicode string) in tokenURI" do - data = %{ - "c87b56dd" => - {:ok, - [ - "data:application/json;base64,eyJkZXNjcmlwdGlvbiI6ICJQdW5rIERvbWFpbnMgZGlnaXRhbCBpZGVudGl0eSDDry4gVmlzaXQgaHR0cHM6Ly9wdW5rLmRvbWFpbnMvIn0=" - ]} - } + data = + {:ok, + [ + "data:application/json;base64,eyJkZXNjcmlwdGlvbiI6ICJQdW5rIERvbWFpbnMgZGlnaXRhbCBpZGVudGl0eSDDry4gVmlzaXQgaHR0cHM6Ly9wdW5rLmRvbWFpbnMvIn0=" + ]} assert MetadataRetriever.fetch_json(data) == {:ok, @@ -389,13 +263,11 @@ defmodule Explorer.Token.MetadataRetrieverTest do Conn.resp(conn, 200, json) end) - data = %{ - "c87b56dd" => - {:ok, - [ - "http://localhost:#{bypass.port}#{path}" - ]} - } + data = + {:ok, + [ + "http://localhost:#{bypass.port}#{path}" + ]} assert {:ok, %{ @@ -418,13 +290,11 @@ defmodule Explorer.Token.MetadataRetrieverTest do Conn.resp(conn, 200, json) end) - data = %{ - "c87b56dd" => - {:ok, - [ - "http://localhost:#{bypass.port}#{path}" - ]} - } + data = + {:ok, + [ + "http://localhost:#{bypass.port}#{path}" + ]} {:ok, %{ @@ -437,13 +307,11 @@ defmodule Explorer.Token.MetadataRetrieverTest do test "Fetches metadata from '${url}'", %{bypass: bypass} do path = "/data/8/8578.json" - data = %{ - "c87b56dd" => - {:ok, - [ - "'http://localhost:#{bypass.port}#{path}'" - ]} - } + data = + {:ok, + [ + "'http://localhost:#{bypass.port}#{path}'" + ]} json = """ { @@ -470,11 +338,9 @@ defmodule Explorer.Token.MetadataRetrieverTest do end test "Process custom execution reverted" do - data = %{ - "c87b56dd" => - {:error, - "(3) execution reverted: Nonexistent token (0x08c379a0000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000114e6f6e6578697374656e7420746f6b656e000000000000000000000000000000)"} - } + data = + {:error, + "(3) execution reverted: Nonexistent token (0x08c379a0000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000114e6f6e6578697374656e7420746f6b656e000000000000000000000000000000)"} assert {:ok, %{error: "VM execution error"}} == MetadataRetriever.fetch_json(data) end @@ -506,7 +372,7 @@ defmodule Explorer.Token.MetadataRetrieverTest do "name" => "asda", "salePrice" => 34 } - }} == MetadataRetriever.fetch_json(%{"0e89341c" => {:ok, [data]}}) + }} == MetadataRetriever.fetch_json({:ok, [data]}) Application.put_env(:explorer, :http_adapter, HTTPoison) end @@ -552,54 +418,7 @@ defmodule Explorer.Token.MetadataRetrieverTest do %{ metadata: Jason.decode!(json) }} == - MetadataRetriever.fetch_json(%{"0e89341c" => {:ok, ["http://localhost:#{bypass.port}#{path}"]}}) - end - - test "fetch ipfs of ipfs/{id} format" do - EthereumJSONRPC.Mox - |> expect(:json_rpc, fn [ - %{ - id: 0, - jsonrpc: "2.0", - method: "eth_call", - params: [ - %{ - data: - "0xc87b56dd0000000000000000000000000000000000000000000000000000000000000000", - to: "0x7e01CC81fCfdf6a71323900288A69e234C464f63" - }, - "latest" - ] - } - ], - _options -> - {:ok, - [ - %{ - id: 0, - jsonrpc: "2.0", - result: - "0x00000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000033697066732f516d6439707654684577676a544262456b4e6d6d47466263704a4b773137666e524241543454643472636f67323200000000000000000000000000" - } - ]} - end) - - Application.put_env(:explorer, :http_adapter, Explorer.Mox.HTTPoison) - - Explorer.Mox.HTTPoison - |> expect(:get, fn "https://ipfs.io/ipfs/Qmd9pvThEwgjTBbEkNmmGFbcpJKw17fnRBAT4Td4rcog22", _headers, _options -> - {:ok, %HTTPoison.Response{status_code: 200, body: "123", headers: [{"Content-Type", "image/jpg"}]}} - end) - - assert {:ok, - %{ - metadata: %{ - "image" => "https://ipfs.io/ipfs/Qmd9pvThEwgjTBbEkNmmGFbcpJKw17fnRBAT4Td4rcog22" - } - }} == - MetadataRetriever.fetch_metadata("0x7e01CC81fCfdf6a71323900288A69e234C464f63", 0) - - Application.put_env(:explorer, :http_adapter, HTTPoison) + MetadataRetriever.fetch_json({:ok, ["http://localhost:#{bypass.port}#{path}"]}) end end end diff --git a/apps/indexer/lib/indexer/fetcher/token_instance/helper.ex b/apps/indexer/lib/indexer/fetcher/token_instance/helper.ex index 8fa628ad9d..e5e2256ff9 100644 --- a/apps/indexer/lib/indexer/fetcher/token_instance/helper.ex +++ b/apps/indexer/lib/indexer/fetcher/token_instance/helper.ex @@ -3,45 +3,171 @@ defmodule Indexer.Fetcher.TokenInstance.Helper do Common functions for Indexer.Fetcher.TokenInstance fetchers """ alias Explorer.Chain - alias Explorer.Chain.{Hash, Token.Instance} + alias Explorer.SmartContract.Reader alias Indexer.Fetcher.TokenInstance.MetadataRetriever - @spec fetch_instance(Hash.Address.t(), Decimal.t() | non_neg_integer()) :: {:ok, Instance.t()} - def fetch_instance(token_contract_address_hash, token_id) do - token_id = prepare_token_id(token_id) - - case MetadataRetriever.fetch_metadata(to_string(token_contract_address_hash), token_id) do - {:ok, %{metadata: metadata}} -> - params = %{ - token_id: token_id, - token_contract_address_hash: token_contract_address_hash, - metadata: metadata, - error: nil + @cryptokitties_address_hash "0x06012c8cf97bead5deae237070f9587f8e7a266d" + + @token_uri "c87b56dd" + @uri "0e89341c" + + @erc_721_1155_abi [ + %{ + "type" => "function", + "stateMutability" => "view", + "payable" => false, + "outputs" => [ + %{"type" => "string", "name" => ""} + ], + "name" => "tokenURI", + "inputs" => [ + %{ + "type" => "uint256", + "name" => "_tokenId" + } + ], + "constant" => true + }, + %{ + "type" => "function", + "stateMutability" => "view", + "payable" => false, + "outputs" => [ + %{ + "type" => "string", + "name" => "", + "internalType" => "string" + } + ], + "name" => "uri", + "inputs" => [ + %{ + "type" => "uint256", + "name" => "_id", + "internalType" => "uint256" } + ], + "constant" => true + } + ] + + @spec batch_fetch_instances([%{}]) :: list() + def batch_fetch_instances(token_instances) do + token_instances = + Enum.map(token_instances, fn + %{contract_address_hash: hash, token_id: token_id} -> {hash, token_id} + {_, _} = tuple -> tuple + end) + + splitted = + Enum.group_by(token_instances, fn {contract_address_hash, _token_id} -> + to_string(contract_address_hash) == @cryptokitties_address_hash + end) - {:ok, _result} = Chain.upsert_token_instance(params) + cryptokitties = + (splitted[true] || []) + |> Enum.map(fn {contract_address_hash, token_id} -> + {{:ok, ["https://api.cryptokitties.co/kitties/{id}"]}, to_string(token_id), contract_address_hash, token_id} + end) - {:ok, %{error: error}} -> - upsert_token_instance_with_error(token_id, token_contract_address_hash, error) + other = splitted[false] || [] - {:error_code, code} -> - upsert_token_instance_with_error(token_id, token_contract_address_hash, "request error: #{code}") + token_types_map = + Enum.reduce(other, %{}, fn {contract_address_hash, _token_id}, acc -> + address_hash_string = to_string(contract_address_hash) - {:error, reason} -> - upsert_token_instance_with_error(token_id, token_contract_address_hash, reason) - end + Map.put_new(acc, address_hash_string, Chain.get_token_type(contract_address_hash)) + end) + + contract_results = + (other + |> Enum.map(fn {contract_address_hash, token_id} -> + token_id = prepare_token_id(token_id) + contract_address_hash_string = to_string(contract_address_hash) + + prepare_request(token_types_map[contract_address_hash_string], contract_address_hash_string, token_id) + end) + |> Reader.query_contracts(@erc_721_1155_abi, [], false) + |> Enum.zip_reduce(other, [], fn result, {contract_address_hash, token_id}, acc -> + token_id = prepare_token_id(token_id) + + [ + {result, normalize_token_id(token_types_map[to_string(contract_address_hash)], token_id), + contract_address_hash, token_id} + | acc + ] + end) + |> Enum.reverse()) ++ + cryptokitties + + contract_results + |> Enum.map(fn {result, normalized_token_id, _contract_address_hash, _token_id} -> + Task.async(fn -> MetadataRetriever.fetch_json(result, normalized_token_id) end) + end) + |> Task.yield_many(:infinity) + |> Enum.zip(contract_results) + |> Enum.map(fn {{_task, res}, {_result, _normalized_token_id, contract_address_hash, token_id}} -> + case res do + {:ok, result} -> + result_to_insert_params(result, contract_address_hash, token_id) + + {:exit, reason} -> + result_to_insert_params( + {:error, MetadataRetriever.truncate_error("Terminated:" <> inspect(reason))}, + contract_address_hash, + token_id + ) + end + end) + |> Chain.upsert_token_instances_list() end defp prepare_token_id(%Decimal{} = token_id), do: Decimal.to_integer(token_id) defp prepare_token_id(token_id), do: token_id - defp upsert_token_instance_with_error(token_id, token_contract_address_hash, error) do - params = %{ + defp prepare_request("ERC-721", contract_address_hash_string, token_id) do + %{ + contract_address: contract_address_hash_string, + method_id: @token_uri, + args: [token_id], + block_number: nil + } + end + + defp prepare_request(_token_type, contract_address_hash_string, token_id) do + %{ + contract_address: contract_address_hash_string, + method_id: @uri, + args: [token_id], + block_number: nil + } + end + + defp normalize_token_id("ERC-721", _token_id), do: nil + + defp normalize_token_id(_token_type, token_id), + do: token_id |> Integer.to_string(16) |> String.downcase() |> String.pad_leading(64, "0") + + defp result_to_insert_params({:ok, %{metadata: metadata}}, token_contract_address_hash, token_id) do + %{ token_id: token_id, token_contract_address_hash: token_contract_address_hash, - error: error + metadata: metadata, + error: nil } + end + + defp result_to_insert_params({:error_code, code}, token_contract_address_hash, token_id), + do: token_instance_map_with_error(token_id, token_contract_address_hash, "request error: #{code}") - {:ok, _result} = Chain.upsert_token_instance(params) + defp result_to_insert_params({:error, reason}, token_contract_address_hash, token_id), + do: token_instance_map_with_error(token_id, token_contract_address_hash, reason) + + defp token_instance_map_with_error(token_id, token_contract_address_hash, error) do + %{ + token_id: token_id, + token_contract_address_hash: token_contract_address_hash, + error: error + } end end diff --git a/apps/indexer/lib/indexer/fetcher/token_instance/metadata_retriever.ex b/apps/indexer/lib/indexer/fetcher/token_instance/metadata_retriever.ex index bb4d39aee9..89c383f5a5 100644 --- a/apps/indexer/lib/indexer/fetcher/token_instance/metadata_retriever.ex +++ b/apps/indexer/lib/indexer/fetcher/token_instance/metadata_retriever.ex @@ -1,6 +1,6 @@ defmodule Indexer.Fetcher.TokenInstance.MetadataRetriever do @moduledoc """ - Fetches ERC721 token instance metadata. + Fetches ERC-721 & ERC-1155 token instance metadata. """ require Logger @@ -8,55 +8,6 @@ defmodule Indexer.Fetcher.TokenInstance.MetadataRetriever do alias Explorer.SmartContract.Reader alias HTTPoison.{Error, Response} - @token_uri "c87b56dd" - - @abi [ - %{ - "type" => "function", - "stateMutability" => "view", - "payable" => false, - "outputs" => [ - %{"type" => "string", "name" => ""} - ], - "name" => "tokenURI", - "inputs" => [ - %{ - "type" => "uint256", - "name" => "_tokenId" - } - ], - "constant" => true - } - ] - - @uri "0e89341c" - - @abi_uri [ - %{ - "type" => "function", - "stateMutability" => "view", - "payable" => false, - "outputs" => [ - %{ - "type" => "string", - "name" => "", - "internalType" => "string" - } - ], - "name" => "uri", - "inputs" => [ - %{ - "type" => "uint256", - "name" => "_id", - "internalType" => "uint256" - } - ], - "constant" => true - } - ] - - @cryptokitties_address_hash "0x06012c8cf97bead5deae237070f9587f8e7a266d" - @no_uri_error "no uri" @vm_execution_error "VM execution error" @ipfs_protocol "ipfs://" @@ -69,54 +20,29 @@ defmodule Indexer.Fetcher.TokenInstance.MetadataRetriever do @ignored_hosts ["localhost", "127.0.0.1", "0.0.0.0", "", nil] - def fetch_metadata(unquote(@cryptokitties_address_hash), token_id) do - %{@token_uri => {:ok, ["https://api.cryptokitties.co/kitties/{id}"]}} - |> fetch_json(to_string(token_id)) - end - - def fetch_metadata(contract_address_hash, token_id) do - # c87b56dd = keccak256(tokenURI(uint256)) - contract_functions = %{@token_uri => [token_id]} - - res = - contract_address_hash - |> query_contract(contract_functions, @abi) - |> fetch_json() - - if res == {:ok, %{error: @vm_execution_error}} do - hex_normalized_token_id = token_id |> Integer.to_string(16) |> String.downcase() |> String.pad_leading(64, "0") - - contract_functions_uri = %{@uri => [token_id]} - - contract_address_hash - |> query_contract(contract_functions_uri, @abi_uri) - |> fetch_json(hex_normalized_token_id) - else - res - end - end - def query_contract(contract_address_hash, contract_functions, abi) do Reader.query_contract(contract_address_hash, abi, contract_functions, false) end + @doc """ + Fetch/parse metadata using smart-contract's response + """ + @spec fetch_json(any, binary() | nil) :: {:error, binary} | {:error_code, any} | {:ok, %{metadata: any}} def fetch_json(uri, hex_token_id \\ nil) - def fetch_json(uri, _hex_token_id) when uri in [%{@token_uri => {:ok, [""]}}, %{@uri => {:ok, [""]}}] do - {:ok, %{error: @no_uri_error}} - end - - def fetch_json(%{@token_uri => uri}, hex_token_id) do - fetch_json_from_uri(uri, hex_token_id) + def fetch_json(uri, _hex_token_id) when uri in [{:ok, [""]}, {:ok, [""]}] do + {:error, @no_uri_error} end - def fetch_json(%{@uri => uri}, hex_token_id) do + def fetch_json(uri, hex_token_id) do fetch_json_from_uri(uri, hex_token_id) end defp fetch_json_from_uri({:error, error}, _hex_token_id) do + error = to_string(error) + if error =~ "execution reverted" or error =~ @vm_execution_error do - {:ok, %{error: @vm_execution_error}} + {:error, @vm_execution_error} else Logger.debug(["Unknown metadata format error #{inspect(error)}."], fetcher: :token_instances) @@ -351,5 +277,9 @@ defmodule Indexer.Fetcher.TokenInstance.MetadataRetriever do String.replace(token_uri, @erc1155_token_id_placeholder, hex_token_id) end - defp truncate_error(error), do: String.slice(error, 0, @max_error_length) + @doc """ + Truncate error string to @max_error_length symbols + """ + @spec truncate_error(binary()) :: binary() + def truncate_error(error), do: String.slice(error, 0, @max_error_length) end diff --git a/apps/indexer/lib/indexer/fetcher/token_instance/realtime.ex b/apps/indexer/lib/indexer/fetcher/token_instance/realtime.ex index 3f82494b55..804c9258c8 100644 --- a/apps/indexer/lib/indexer/fetcher/token_instance/realtime.ex +++ b/apps/indexer/lib/indexer/fetcher/token_instance/realtime.ex @@ -32,10 +32,12 @@ defmodule Indexer.Fetcher.TokenInstance.Realtime do end @impl BufferedTask - def run([%{contract_address_hash: hash, token_id: token_id}], _json_rpc_named_arguments) do - if not Chain.token_instance_exists?(token_id, hash) do - fetch_instance(hash, token_id) - end + def run(token_instances, _) when is_list(token_instances) do + token_instances + |> Enum.filter(fn %{contract_address_hash: hash, token_id: token_id} -> + not Chain.token_instance_exists?(token_id, hash) + end) + |> batch_fetch_instances() :ok end @@ -75,7 +77,7 @@ defmodule Indexer.Fetcher.TokenInstance.Realtime do [ flush_interval: 100, max_concurrency: Application.get_env(:indexer, __MODULE__)[:concurrency] || @default_max_concurrency, - max_batch_size: @default_max_batch_size, + max_batch_size: Application.get_env(:indexer, __MODULE__)[:batch_size] || @default_max_batch_size, poll: false, task_supervisor: __MODULE__.TaskSupervisor ] diff --git a/apps/indexer/lib/indexer/fetcher/token_instance/retry.ex b/apps/indexer/lib/indexer/fetcher/token_instance/retry.ex index 24b2df433f..a09bce4596 100644 --- a/apps/indexer/lib/indexer/fetcher/token_instance/retry.ex +++ b/apps/indexer/lib/indexer/fetcher/token_instance/retry.ex @@ -13,7 +13,7 @@ defmodule Indexer.Fetcher.TokenInstance.Retry do @behaviour BufferedTask - @default_max_batch_size 1 + @default_max_batch_size 10 @default_max_concurrency 10 @doc false @@ -40,14 +40,16 @@ defmodule Indexer.Fetcher.TokenInstance.Retry do end @impl BufferedTask - def run([%{contract_address_hash: hash, token_id: token_id, updated_at: updated_at}], _json_rpc_named_arguments) do + def run(token_instances, _json_rpc_named_arguments) when is_list(token_instances) do refetch_interval = Application.get_env(:indexer, __MODULE__)[:refetch_interval] - if updated_at - |> DateTime.add(refetch_interval, :millisecond) - |> DateTime.compare(DateTime.utc_now()) != :gt do - fetch_instance(hash, token_id) - end + token_instances + |> Enum.filter(fn %{contract_address_hash: _hash, token_id: _token_id, updated_at: updated_at} -> + updated_at + |> DateTime.add(refetch_interval, :millisecond) + |> DateTime.compare(DateTime.utc_now()) != :gt + end) + |> batch_fetch_instances() :ok end @@ -56,7 +58,7 @@ defmodule Indexer.Fetcher.TokenInstance.Retry do [ flush_interval: :timer.minutes(10), max_concurrency: Application.get_env(:indexer, __MODULE__)[:concurrency] || @default_max_concurrency, - max_batch_size: @default_max_batch_size, + max_batch_size: Application.get_env(:indexer, __MODULE__)[:batch_size] || @default_max_batch_size, task_supervisor: __MODULE__.TaskSupervisor ] end diff --git a/apps/indexer/lib/indexer/fetcher/token_instance/sanitize.ex b/apps/indexer/lib/indexer/fetcher/token_instance/sanitize.ex index 31c594ec00..1bc3a70bd1 100644 --- a/apps/indexer/lib/indexer/fetcher/token_instance/sanitize.ex +++ b/apps/indexer/lib/indexer/fetcher/token_instance/sanitize.ex @@ -13,7 +13,7 @@ defmodule Indexer.Fetcher.TokenInstance.Sanitize do @behaviour BufferedTask - @default_max_batch_size 1 + @default_max_batch_size 10 @default_max_concurrency 10 @doc false def child_spec([init_options, gen_server_options]) do @@ -36,10 +36,12 @@ defmodule Indexer.Fetcher.TokenInstance.Sanitize do end @impl BufferedTask - def run([%{contract_address_hash: hash, token_id: token_id}], _json_rpc_named_arguments) do - if not Chain.token_instance_exists?(token_id, hash) do - fetch_instance(hash, token_id) - end + def run(token_instances, _) when is_list(token_instances) do + token_instances + |> Enum.filter(fn %{contract_address_hash: hash, token_id: token_id} -> + not Chain.token_instance_exists?(token_id, hash) + end) + |> batch_fetch_instances() :ok end @@ -48,7 +50,7 @@ defmodule Indexer.Fetcher.TokenInstance.Sanitize do [ flush_interval: :infinity, max_concurrency: Application.get_env(:indexer, __MODULE__)[:concurrency] || @default_max_concurrency, - max_batch_size: @default_max_batch_size, + max_batch_size: Application.get_env(:indexer, __MODULE__)[:batch_size] || @default_max_batch_size, poll: false, task_supervisor: __MODULE__.TaskSupervisor ] diff --git a/apps/indexer/test/indexer/fetcher/token_instance/helper_test.exs b/apps/indexer/test/indexer/fetcher/token_instance/helper_test.exs new file mode 100644 index 0000000000..8d9987f9de --- /dev/null +++ b/apps/indexer/test/indexer/fetcher/token_instance/helper_test.exs @@ -0,0 +1,178 @@ +defmodule Indexer.Fetcher.TokenInstance.HelperTest do + use EthereumJSONRPC.Case + use Explorer.DataCase + + alias Explorer.Chain.Token.Instance + alias EthereumJSONRPC.Encoder + alias Indexer.Fetcher.TokenInstance.Helper + alias Plug.Conn + + import Mox + + setup :verify_on_exit! + setup :set_mox_global + + setup do + bypass = Bypass.open() + + {:ok, bypass: bypass} + end + + describe "fetch instance tests" do + test "fetches json metadata for kitties" do + Application.put_env(:explorer, :http_adapter, Explorer.Mox.HTTPoison) + + result = + "{\"id\":100500,\"name\":\"KittyBlue_2_Lemonade\",\"generation\":20,\"genes\":\"623509754227292470437941473598751240781530569131665917719736997423495595\",\"created_at\":\"2017-12-06T01:56:27.000Z\",\"birthday\":\"2017-12-06T00:00:00.000Z\",\"image_url\":\"https://img.cryptokitties.co/0x06012c8cf97bead5deae237070f9587f8e7a266d/100500.svg\",\"image_url_cdn\":\"https://img.cn.cryptokitties.co/0x06012c8cf97bead5deae237070f9587f8e7a266d/100500.svg\",\"color\":\"strawberry\",\"background_color\":\"#ffe0e5\",\"bio\":\"Shalom! I'm KittyBlue_2_Lemonade. I'm a professional Foreign Film Director and I love cantaloupe. I'm convinced that the world is flat. One day I'll prove it. It's pawesome to meet you!\",\"kitty_type\":null,\"is_fancy\":false,\"is_exclusive\":false,\"is_special_edition\":false,\"fancy_type\":null,\"language\":\"en\",\"is_prestige\":false,\"prestige_type\":null,\"prestige_ranking\":null,\"prestige_time_limit\":null,\"status\":{\"is_ready\":true,\"is_gestating\":false,\"cooldown\":1410310201506,\"dynamic_cooldown\":1475064986478,\"cooldown_index\":10,\"cooldown_end_block\":0,\"pending_tx_type\":null,\"pending_tx_since\":null},\"purrs\":{\"count\":1,\"is_purred\":false},\"watchlist\":{\"count\":0,\"is_watchlisted\":false},\"hatcher\":{\"address\":\"0x7b9ea9ac69b8fde875554321472c732eeff06ca0\",\"image\":\"14\",\"nickname\":\"KittyBlu\",\"hasDapper\":false,\"twitter_id\":null,\"twitter_image_url\":null,\"twitter_handle\":null},\"auction\":{},\"offer\":{},\"owner\":{\"address\":\"0x7b9ea9ac69b8fde875554321472c732eeff06ca0\",\"hasDapper\":false,\"twitter_id\":null,\"twitter_image_url\":null,\"twitter_handle\":null,\"image\":\"14\",\"nickname\":\"KittyBlu\"},\"matron\":{\"id\":46234,\"name\":\"KittyBlue_1_Limegreen\",\"generation\":10,\"enhanced_cattributes\":[{\"type\":\"body\",\"kittyId\":19631,\"position\":105,\"description\":\"cymric\"},{\"type\":\"coloreyes\",\"kittyId\":40356,\"position\":263,\"description\":\"limegreen\"},{\"type\":\"eyes\",\"kittyId\":3185,\"position\":16,\"description\":\"raisedbrow\"},{\"type\":\"pattern\",\"kittyId\":46234,\"position\":-1,\"description\":\"totesbasic\"},{\"type\":\"mouth\",\"kittyId\":46234,\"position\":-1,\"description\":\"happygokitty\"},{\"type\":\"colorprimary\",\"kittyId\":46234,\"position\":-1,\"description\":\"greymatter\"},{\"type\":\"colorsecondary\",\"kittyId\":46234,\"position\":-1,\"description\":\"lemonade\"},{\"type\":\"colortertiary\",\"kittyId\":46234,\"position\":-1,\"description\":\"granitegrey\"}],\"owner_wallet_address\":\"0x7b9ea9ac69b8fde875554321472c732eeff06ca0\",\"owner\":{\"address\":\"0x7b9ea9ac69b8fde875554321472c732eeff06ca0\"},\"created_at\":\"2017-12-03T21:29:17.000Z\",\"image_url\":\"https://img.cryptokitties.co/0x06012c8cf97bead5deae237070f9587f8e7a266d/46234.svg\",\"image_url_cdn\":\"https://img.cn.cryptokitties.co/0x06012c8cf97bead5deae237070f9587f8e7a266d/46234.svg\",\"color\":\"limegreen\",\"is_fancy\":false,\"kitty_type\":null,\"is_exclusive\":false,\"is_special_edition\":false,\"fancy_type\":null,\"status\":{\"is_ready\":true,\"is_gestating\":false,\"cooldown\":1486487069384},\"hatched\":true,\"wrapped\":false,\"image_url_png\":\"https://img.cryptokitties.co/0x06012c8cf97bead5deae237070f9587f8e7a266d/46234.png\"},\"sire\":{\"id\":82090,\"name\":null,\"generation\":19,\"enhanced_cattributes\":[{\"type\":\"body\",\"kittyId\":82090,\"position\":-1,\"description\":\"himalayan\"},{\"type\":\"coloreyes\",\"kittyId\":82090,\"position\":-1,\"description\":\"strawberry\"},{\"type\":\"eyes\",\"kittyId\":82090,\"position\":-1,\"description\":\"thicccbrowz\"},{\"type\":\"pattern\",\"kittyId\":82090,\"position\":-1,\"description\":\"totesbasic\"},{\"type\":\"mouth\",\"kittyId\":82090,\"position\":-1,\"description\":\"pouty\"},{\"type\":\"colorprimary\",\"kittyId\":82090,\"position\":-1,\"description\":\"aquamarine\"},{\"type\":\"colorsecondary\",\"kittyId\":82090,\"position\":-1,\"description\":\"chocolate\"},{\"type\":\"colortertiary\",\"kittyId\":82090,\"position\":-1,\"description\":\"granitegrey\"}],\"owner_wallet_address\":\"0x798fdad0cedc4b298fc7d53a982fa0c5f447eaa5\",\"owner\":{\"address\":\"0x798fdad0cedc4b298fc7d53a982fa0c5f447eaa5\"},\"created_at\":\"2017-12-05T06:30:05.000Z\",\"image_url\":\"https://img.cryptokitties.co/0x06012c8cf97bead5deae237070f9587f8e7a266d/82090.svg\",\"image_url_cdn\":\"https://img.cn.cryptokitties.co/0x06012c8cf97bead5deae237070f9587f8e7a266d/82090.svg\",\"color\":\"strawberry\",\"is_fancy\":false,\"is_exclusive\":false,\"is_special_edition\":false,\"fancy_type\":null,\"status\":{\"is_ready\":true,\"is_gestating\":false,\"cooldown\":1486619010030},\"kitty_type\":null,\"hatched\":true,\"wrapped\":false,\"image_url_png\":\"https://img.cryptokitties.co/0x06012c8cf97bead5deae237070f9587f8e7a266d/82090.png\"},\"children\":[],\"hatched\":true,\"wrapped\":false,\"enhanced_cattributes\":[{\"type\":\"colorprimary\",\"description\":\"greymatter\",\"position\":null,\"kittyId\":100500},{\"type\":\"coloreyes\",\"description\":\"strawberry\",\"position\":null,\"kittyId\":100500},{\"type\":\"body\",\"description\":\"himalayan\",\"position\":null,\"kittyId\":100500},{\"type\":\"colorsecondary\",\"description\":\"lemonade\",\"position\":null,\"kittyId\":100500},{\"type\":\"mouth\",\"description\":\"pouty\",\"position\":null,\"kittyId\":100500},{\"type\":\"pattern\",\"description\":\"totesbasic\",\"position\":null,\"kittyId\":100500},{\"type\":\"eyes\",\"description\":\"thicccbrowz\",\"position\":null,\"kittyId\":100500},{\"type\":\"colortertiary\",\"description\":\"kittencream\",\"position\":null,\"kittyId\":100500},{\"type\":\"secret\",\"description\":\"se5\",\"position\":-1,\"kittyId\":100500},{\"type\":\"purrstige\",\"description\":\"pu20\",\"position\":-1,\"kittyId\":100500}],\"variation\":null,\"variation_ranking\":null,\"image_url_png\":\"https://img.cryptokitties.co/0x06012c8cf97bead5deae237070f9587f8e7a266d/100500.png\",\"items\":[]}" + + Explorer.Mox.HTTPoison + |> expect(:get, fn "https://api.cryptokitties.co/kitties/100500", _headers, _options -> + {:ok, %HTTPoison.Response{status_code: 200, body: result}} + end) + + insert(:token, + contract_address: build(:address, hash: "0x06012c8cf97bead5deae237070f9587f8e7a266d"), + type: "ERC-721" + ) + + [{:ok, %Instance{metadata: metadata}}] = + Helper.batch_fetch_instances([{"0x06012c8cf97bead5deae237070f9587f8e7a266d", 100_500}]) + + assert Map.get(metadata, "name") == "KittyBlue_2_Lemonade" + + Application.put_env(:explorer, :http_adapter, HTTPoison) + end + + test "replace {id} with actual token_id", %{bypass: bypass} do + json = """ + { + "name": "Sérgio Mendonça {id}" + } + """ + + abi = + [ + %{ + "type" => "function", + "stateMutability" => "nonpayable", + "payable" => false, + "outputs" => [], + "name" => "tokenURI", + "inputs" => [ + %{"type" => "string", "name" => "name", "internalType" => "string"} + ] + } + ] + |> ABI.parse_specification() + |> Enum.at(0) + + encoded_url = + abi + |> Encoder.encode_function_call(["http://localhost:#{bypass.port}/api/card/{id}"]) + |> String.replace("4cf12d26", "") + + EthereumJSONRPC.Mox + |> expect(:json_rpc, fn [ + %{ + id: 0, + jsonrpc: "2.0", + method: "eth_call", + params: [ + %{ + data: + "0x0e89341c0000000000000000000000000000000000000000000000000000000000000309", + to: "0x5caebd3b32e210e85ce3e9d51638b9c445481567" + }, + "latest" + ] + } + ], + _options -> + {:ok, + [ + %{ + id: 0, + jsonrpc: "2.0", + result: encoded_url + } + ]} + end) + + Bypass.expect( + bypass, + "GET", + "/api/card/0000000000000000000000000000000000000000000000000000000000000309", + fn conn -> + Conn.resp(conn, 200, json) + end + ) + + insert(:token, + contract_address: build(:address, hash: "0x5caebd3b32e210e85ce3e9d51638b9c445481567"), + type: "ERC-1155" + ) + + assert [ + {:ok, + %Instance{ + metadata: %{ + "name" => "Sérgio Mendonça 0000000000000000000000000000000000000000000000000000000000000309" + } + }} + ] = Helper.batch_fetch_instances([{"0x5caebd3b32e210e85ce3e9d51638b9c445481567", 777}]) + end + + test "fetch ipfs of ipfs/{id} format" do + EthereumJSONRPC.Mox + |> expect(:json_rpc, fn [ + %{ + id: 0, + jsonrpc: "2.0", + method: "eth_call", + params: [ + %{ + data: + "0xc87b56dd0000000000000000000000000000000000000000000000000000000000000000", + to: "0x7e01CC81fCfdf6a71323900288A69e234C464f63" + }, + "latest" + ] + } + ], + _options -> + {:ok, + [ + %{ + id: 0, + jsonrpc: "2.0", + result: + "0x00000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000033697066732f516d6439707654684577676a544262456b4e6d6d47466263704a4b773137666e524241543454643472636f67323200000000000000000000000000" + } + ]} + end) + + Application.put_env(:explorer, :http_adapter, Explorer.Mox.HTTPoison) + + Explorer.Mox.HTTPoison + |> expect(:get, fn "https://ipfs.io/ipfs/Qmd9pvThEwgjTBbEkNmmGFbcpJKw17fnRBAT4Td4rcog22", _headers, _options -> + {:ok, %HTTPoison.Response{status_code: 200, body: "123", headers: [{"Content-Type", "image/jpg"}]}} + end) + + insert(:token, + contract_address: build(:address, hash: "0x7e01CC81fCfdf6a71323900288A69e234C464f63"), + type: "ERC-721" + ) + + assert [ + {:ok, + %Instance{ + metadata: %{ + "image" => "https://ipfs.io/ipfs/Qmd9pvThEwgjTBbEkNmmGFbcpJKw17fnRBAT4Td4rcog22" + } + }} + ] = Helper.batch_fetch_instances([{"0x7e01CC81fCfdf6a71323900288A69e234C464f63", 0}]) + + Application.put_env(:explorer, :http_adapter, HTTPoison) + end + end +end diff --git a/apps/indexer/test/test_helper.exs b/apps/indexer/test/test_helper.exs index 0df4333702..381b6272bc 100644 --- a/apps/indexer/test/test_helper.exs +++ b/apps/indexer/test/test_helper.exs @@ -17,6 +17,7 @@ end Mox.defmock(EthereumJSONRPC.Mox, for: EthereumJSONRPC.Transport) Mox.defmock(Indexer.BufferedTaskTest.RetryableTask, for: Indexer.BufferedTask) Mox.defmock(Indexer.BufferedTaskTest.ShrinkableTask, for: Indexer.BufferedTask) +Mox.defmock(Explorer.Mox.HTTPoison, for: HTTPoison.Base) ExUnit.configure(formatters: [JUnitFormatter, ExUnit.CLIFormatter]) ExUnit.start() diff --git a/config/runtime.exs b/config/runtime.exs index bc104efce1..10831b7a67 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -507,13 +507,16 @@ config :indexer, Indexer.Fetcher.BlockReward, config :indexer, Indexer.Fetcher.TokenInstance.Retry, concurrency: ConfigHelper.parse_integer_env_var("INDEXER_TOKEN_INSTANCE_RETRY_CONCURRENCY", 10), + batch_size: ConfigHelper.parse_integer_env_var("INDEXER_TOKEN_INSTANCE_RETRY_BATCH_SIZE", 10), refetch_interval: ConfigHelper.parse_time_env_var("INDEXER_TOKEN_INSTANCE_RETRY_REFETCH_INTERVAL", "24h") config :indexer, Indexer.Fetcher.TokenInstance.Realtime, - concurrency: ConfigHelper.parse_integer_env_var("INDEXER_TOKEN_INSTANCE_REALTIME_CONCURRENCY", 10) + concurrency: ConfigHelper.parse_integer_env_var("INDEXER_TOKEN_INSTANCE_REALTIME_CONCURRENCY", 10), + batch_size: ConfigHelper.parse_integer_env_var("INDEXER_TOKEN_INSTANCE_REALTIME_BATCH_SIZE", 1) config :indexer, Indexer.Fetcher.TokenInstance.Sanitize, - concurrency: ConfigHelper.parse_integer_env_var("INDEXER_TOKEN_INSTANCE_SANITIZE_CONCURRENCY", 10) + concurrency: ConfigHelper.parse_integer_env_var("INDEXER_TOKEN_INSTANCE_SANITIZE_CONCURRENCY", 10), + batch_size: ConfigHelper.parse_integer_env_var("INDEXER_TOKEN_INSTANCE_SANITIZE_BATCH_SIZE", 10) config :indexer, Indexer.Fetcher.InternalTransaction, batch_size: ConfigHelper.parse_integer_env_var("INDEXER_INTERNAL_TRANSACTIONS_BATCH_SIZE", 10), diff --git a/docker-compose/envs/common-blockscout.env b/docker-compose/envs/common-blockscout.env index 65b40fd43f..7fcb5879a0 100644 --- a/docker-compose/envs/common-blockscout.env +++ b/docker-compose/envs/common-blockscout.env @@ -211,3 +211,6 @@ ACCOUNT_REDIS_URL=redis://redis_db:6379 EIP_1559_ELASTICITY_MULTIPLIER=2 # API_SENSITIVE_ENDPOINTS_KEY= # ACCOUNT_VERIFICATION_EMAIL_RESEND_INTERVAL= +# INDEXER_TOKEN_INSTANCE_RETRY_BATCH_SIZE=10 +# INDEXER_TOKEN_INSTANCE_REALTIME_BATCH_SIZE=1 +# INDEXER_TOKEN_INSTANCE_SANITIZE_BATCH_SIZE=10 diff --git a/docker/Makefile b/docker/Makefile index 3f948c7106..2067f59b54 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -732,6 +732,15 @@ endif ifdef ACCOUNT_VERIFICATION_EMAIL_RESEND_INTERVAL BLOCKSCOUT_CONTAINER_PARAMS += -e 'ACCOUNT_VERIFICATION_EMAIL_RESEND_INTERVAL=$(ACCOUNT_VERIFICATION_EMAIL_RESEND_INTERVAL)' endif +ifdef INDEXER_TOKEN_INSTANCE_RETRY_BATCH_SIZE + BLOCKSCOUT_CONTAINER_PARAMS += -e 'INDEXER_TOKEN_INSTANCE_RETRY_BATCH_SIZE=$(INDEXER_TOKEN_INSTANCE_RETRY_BATCH_SIZE)' +endif +ifdef INDEXER_TOKEN_INSTANCE_REALTIME_BATCH_SIZE + BLOCKSCOUT_CONTAINER_PARAMS += -e 'INDEXER_TOKEN_INSTANCE_REALTIME_BATCH_SIZE=$(INDEXER_TOKEN_INSTANCE_REALTIME_BATCH_SIZE)' +endif +ifdef INDEXER_TOKEN_INSTANCE_SANITIZE_BATCH_SIZE + BLOCKSCOUT_CONTAINER_PARAMS += -e 'INDEXER_TOKEN_INSTANCE_SANITIZE_BATCH_SIZE=$(INDEXER_TOKEN_INSTANCE_SANITIZE_BATCH_SIZE)' +endif HAS_BLOCKSCOUT_IMAGE := $(shell docker images | grep -sw "${BS_CONTAINER_IMAGE} ") build: