Merge branch 'master' into ab-optimize-token-token-transfers-query

.circleci/config.yml

@@ -39,9 +39,6 @@ jobs:
 
       - run: mix deps.get
 
-      - run:
-          command: sed -i '68,68 s/^/%/' ./deps/hackney/src/hackney_ssl.erl
-
       - restore_cache:
           keys:
             - v7-npm-install-{{ .Branch }}-{{ checksum "apps/block_scout_web/assets/package-lock.json" }}

CHANGELOG.md

@@ -1,6 +1,7 @@
 ## Current
 
 ### Features
+- [#2772](https://github.com/poanetwork/blockscout/pull/2772) - add token instance images to the token inventory tab
 - [#2733](https://github.com/poanetwork/blockscout/pull/2733) - Add cache for first page of uncles
 - [#2735](https://github.com/poanetwork/blockscout/pull/2735) - Add pending transactions cache
 - [#2726](https://github.com/poanetwork/blockscout/pull/2726) - Remove internal_transaction block_number setting from blocks runner
@@ -13,9 +14,16 @@
 - [#2665](https://github.com/poanetwork/blockscout/pull/2665) - new menu layout for mobile devices
 - [#2663](https://github.com/poanetwork/blockscout/pull/2663) - Fetch address counters in parallel
 - [#2642](https://github.com/poanetwork/blockscout/pull/2642) - add ERC721 coin instance page
+- [#2762](https://github.com/poanetwork/blockscout/pull/2762) - on-fly fetching of token instances
+- [#2470](https://github.com/poanetwork/blockscout/pull/2470) - Allow Realtime Fetcher to wait for small skips
 
 ### Fixes
+- [#2770](https://github.com/poanetwork/blockscout/pull/2770) - do not re-fetch token instances without uris
 - [#2769](https://github.com/poanetwork/blockscout/pull/2769) - optimize token token transfers query
+- [#2761](https://github.com/poanetwork/blockscout/pull/2761) - add indexes for token instances fetching queries
+- [#2767](https://github.com/poanetwork/blockscout/pull/2767) - fix websocket subscriptions with token instances
+- [#2765](https://github.com/poanetwork/blockscout/pull/2765) - fixed width issue for cards in mobile view for Transaction Details page
+- [#2755](https://github.com/poanetwork/blockscout/pull/2755) - various token instance fetcher fixes
 - [#2753](https://github.com/poanetwork/blockscout/pull/2753) - fix nft token instance images
 - [#2750](https://github.com/poanetwork/blockscout/pull/2750) - fixed contract buttons color for NFT token instance on each theme
 - [#2746](https://github.com/poanetwork/blockscout/pull/2746) - fixed wrong alignment in logs decoded view
@@ -43,6 +51,7 @@ fixed menu hovers in dark mode desktop view
 - [#2738](https://github.com/poanetwork/blockscout/pull/2738) - do not fail block `internal_transactions_indexed_at` field update
 
 ### Chore
+- [#2749](https://github.com/poanetwork/blockscout/pull/2749) - fix opt 22.1 support
 - [#2724](https://github.com/poanetwork/blockscout/pull/2724) - fix ci by commenting a line in hackney library
 - [#2708](https://github.com/poanetwork/blockscout/pull/2708) - add log index to logs view
 - [#2723](https://github.com/poanetwork/blockscout/pull/2723) - get rid of ex_json_schema warnings

README.md

@@ -33,7 +33,7 @@ Currently available full-featured block explorers (Etherscan, Etherchain, Blockc
 | [Callisto](https://blockscout.com/callisto/mainnet) | [Goerli Testnet](https://blockscout.com/eth/goerli) | [ARTIS](https://explorer.sigma1.artis.network) | [Celo Testnet](https://alfajores-blockscout.celo-testnet.org/) |
 | [Ethereum Classic](https://blockscout.com/etc/mainnet) | [Kovan Testnet](https://blockscout.com/eth/kovan) | [Ether-1](https://blocks.ether1.wattpool.net/) | [Matic Testnet](https://explorer.testnet2.matic.network/) |
 | [Ethereum Mainnet](https://blockscout.com/eth/mainnet) | [LUKSO L14 Testnet](https://blockscout.com/lukso/l14) | [Kotti Testnet](https://kottiexplorer.ethernode.io/) | [Mordor Testnet](https://mordorexplorer.ethernode.io/) |
-| [POA Core Network](https://blockscout.com/poa/core) | [POA Sokol Testnet](https://blockscout.com/poa/sokol) | [Fuse Network](https://explorer.fuse.io/) |  |
+| [POA Core Network](https://blockscout.com/poa/core)| [POA Sokol Testnet](https://blockscout.com/poa/sokol) | [Fuse Network](https://explorer.fuse.io/)  | [Athereum Testnet](http://athexplorer.ava.network/) |
 | [RSK](https://blockscout.com/rsk/mainnet) | [Rinkeby Testnet](https://blockscout.com/eth/rinkeby) | [Oasis Labs](https://blockexplorer.oasiscloud.io/) |  |
 | [xDai Chain](https://blockscout.com/poa/dai) | [Ropsten Testnet](https://blockscout.com/eth/ropsten) | [Petrichor](https://explorer.petrachor.com/) |  |
 |  |  | [PIRL](http://pirl.es/) |  |

apps/block_scout_web/assets/css/app.scss

@@ -116,6 +116,7 @@ $fa-font-path: "~@fortawesome/fontawesome-free/webfonts";
 @import "components/btn_no_border";
 @import "components/custom_tooltips_block_details";
 @import "components/_erc721_token_image_container";
+@import "components/_inventory_token_instance_image_container";
 
 @import "theme/dark-theme";
 

apps/block_scout_web/assets/css/components/_card.scss

@@ -34,7 +34,9 @@ $card-tab-icon-color-active: #fff !default;
 .card-background-1 {
   background-color: $card-background-1;
   color: $card-background-1-text-color;
-
+  @include media-breakpoint-down(sm) {
+    margin-left: 15px;
+  }
   a:not(.dropdown-item),
   a:not(.dropdown-item):hover {
     color: $card-background-1-text-color;
@@ -249,3 +251,9 @@ $card-tab-icon-color-active: #fff !default;
     }
   }
 }
+
+.mob-transaction {
+  @include media-breakpoint-down(sm) {
+    margin-left: 15px!important;
+  } 
+}

apps/block_scout_web/assets/css/components/_inventory_token_instance_image_container.scss

@@ -0,0 +1,10 @@
+/* ERC721 image block */
+.inventory-erc721-image {
+    display: flex;
+    justify-content: center;
+}
+
+ .inventory-erc721-image img {
+    height: 50px;
+}
+/* ERC721 image block end */

apps/block_scout_web/lib/block_scout_web/templates/tokens/inventory/_token.html.eex

@@ -24,5 +24,12 @@
         </span>
       </span>
     </div>
+
+    <div class="pl-5 col-md-2 d-flex flex-column align-items-left justify-content-start justify-content-lg-center">
+      <!-- substitute the span with <img class="tile-image" /> to token with images -->
+      <div class="inventory-erc721-image" >
+        <img src=<%=image_src(@token_transfer.instance)%> />
+      </div>
+    </div>
   </div>
 </div>

apps/block_scout_web/lib/block_scout_web/templates/transaction/_tile.html.eex

@@ -38,11 +38,11 @@
         <div class="d-flex flex-column mt-2">
           <% [first_token_transfer | remaining_token_transfers] = @transaction.token_transfers %>
 
-          <%= render "_token_transfer.html", address: assigns[:current_address], token_transfer: first_token_transfer, conn: @conn %>
+          <%= render "_token_transfer.html", address: assigns[:current_address], token_transfer: first_token_transfer %>
 
           <div class="collapse token-transfer-toggle" id="transaction-<%= @transaction.hash %>">
             <%= for token_transfer <- remaining_token_transfers do %>
-              <%= render "_token_transfer.html", address: assigns[:current_address], token_transfer: token_transfer, conn: @conn %>
+              <%= render "_token_transfer.html", address: assigns[:current_address], token_transfer: token_transfer %>
             <% end %>
           </div>
         </div>

apps/block_scout_web/lib/block_scout_web/templates/transaction/_token_transfer.html.eex

@@ -22,7 +22,7 @@
   <span class="col-xs-12 col-lg-4 ml-3 ml-sm-0 text-truncate">
     <%= case token_transfer_amount(@token_transfer) do %>
       <% {:ok, :erc721_instance} -> %>
-        <%= "TokenID ["%><%= link(@token_transfer.token_id, to: token_instance_path(@conn, :show,  @token_transfer.token.contract_address_hash, to_string(@token_transfer.token_id))) %><%= "] " %>
+        <%= "TokenID ["%><%= link(@token_transfer.token_id, to: token_instance_path(BlockScoutWeb.Endpoint, :show,  @token_transfer.token.contract_address_hash, to_string(@token_transfer.token_id))) %><%= "] " %>
       <% {:ok, value} -> %>
         <%= "#{value} " %>
     <% end %>

apps/block_scout_web/lib/block_scout_web/templates/transaction/overview.html.eex

@@ -227,7 +227,7 @@
       </div>
     <% end %>
       <!-- Gas -->
-      <div class="card flex-grow-1 ml-0 ml-md-5 ml-lg-0">
+      <div class="card flex-grow-1 ml-0 ml-md-5 ml-lg-0 mob-transaction">
         <div class="card-body card-body-flex-column-space-between">
           <h2 class="card-title balance-card-title"> <%= gettext "Gas" %> </h2>
           <div class="text-right">

apps/block_scout_web/lib/block_scout_web/views/tokens/inventory_view.ex

@@ -1,5 +1,7 @@
 defmodule BlockScoutWeb.Tokens.InventoryView do
   use BlockScoutWeb, :view
 
+  import BlockScoutWeb.Tokens.Instance.OverviewView, only: [image_src: 1]
+
   alias BlockScoutWeb.Tokens.OverviewView
 end

apps/ethereum_jsonrpc/mix.exs

@@ -90,7 +90,8 @@ defmodule EthereumJsonrpc.MixProject do
       # `EthereumJSONRPC.WebSocket`
       {:websocket_client, "~> 1.3"},
       {:decimal, "~> 1.0"},
-      {:decorator, "~> 1.2"}
+      {:decorator, "~> 1.2"},
+      {:hackney, "~> 1.15.2"}
     ]
   end
 end

apps/explorer/lib/explorer/chain.ex

@@ -2945,8 +2945,10 @@ defmodule Explorer.Chain do
         inner_join: token in Token,
         on: token.contract_address_hash == token_transfer.token_contract_address_hash,
         left_join: instance in Instance,
-        on: token_transfer.token_id == instance.token_id,
-        where: token.type == ^"ERC-721" and is_nil(instance.token_id),
+        on:
+          token_transfer.token_id == instance.token_id and
+            token_transfer.token_contract_address_hash == instance.token_contract_address_hash,
+        where: token.type == ^"ERC-721" and is_nil(instance.token_id) and not is_nil(token_transfer.token_id),
         distinct: [token_transfer.token_contract_address_hash, token_transfer.token_id],
         select: %{contract_address_hash: token_transfer.token_contract_address_hash, token_id: token_transfer.token_id}
       )

apps/explorer/lib/explorer/chain/token/instance.ex

@@ -12,18 +12,21 @@ defmodule Explorer.Chain.Token.Instance do
   * `token_id` - ID of the token
   * `token_contract_address_hash` - Address hash foreign key
   * `metadata` - Token instance metadata
+  * `error` - error fetching token instance
   """
 
   @type t :: %Instance{
           token_id: non_neg_integer(),
           token_contract_address_hash: Hash.Address.t(),
-          metadata: Map.t()
+          metadata: Map.t(),
+          error: String.t()
         }
 
   @primary_key false
   schema "token_instances" do
     field(:token_id, :decimal, primary_key: true)
     field(:metadata, :map)
+    field(:error, :string)
 
     belongs_to(
       :token,
@@ -39,7 +42,7 @@ defmodule Explorer.Chain.Token.Instance do
 
   def changeset(%Instance{} = instance, params \\ %{}) do
     instance
-    |> cast(params, [:token_id, :metadata, :token_contract_address_hash])
+    |> cast(params, [:token_id, :metadata, :token_contract_address_hash, :error])
     |> validate_required([:token_id, :token_contract_address_hash])
     |> foreign_key_constraint(:token_contract_address_hash)
   end

apps/explorer/lib/explorer/chain/token_transfer.ex

@@ -278,11 +278,13 @@ defmodule Explorer.Chain.TokenTransfer do
   def address_to_unique_tokens(contract_address_hash) do
     from(
       tt in TokenTransfer,
-      where: tt.token_contract_address_hash == ^contract_address_hash,
+      left_join: instance in Instance,
+      on: tt.token_contract_address_hash == instance.token_contract_address_hash and tt.token_id == instance.token_id,
+      where: tt.token_contract_address_hash == ^contract_address_hash and tt.token_id == tt.token_id,
       order_by: [desc: tt.block_number],
       distinct: tt.token_id,
       preload: [:to_address],
-      select: tt
+      select: %{tt | instance: instance}
     )
   end
 end

apps/explorer/lib/explorer/token/instance_metadata_retriever.ex

@@ -29,6 +29,8 @@ defmodule Explorer.Token.InstanceMetadataRetriever do
 
   @cryptokitties_address_hash "0x06012c8cf97bead5deae237070f9587f8e7a266d"
 
+  @no_uri_error "no uri"
+
   def fetch_metadata(unquote(@cryptokitties_address_hash), token_id) do
     %{"tokenURI" => {:ok, ["https://api.cryptokitties.co/kitties/#{token_id}"]}}
     |> fetch_json()
@@ -47,13 +49,42 @@ defmodule Explorer.Token.InstanceMetadataRetriever do
   end
 
   defp fetch_json(%{"tokenURI" => {:ok, [""]}}) do
-    {:error, :no_uri}
+    {:ok, %{error: @no_uri_error}}
+  end
+
+  defp fetch_json(%{"tokenURI" => {:error, "(-32015) VM execution error."}}) do
+    {:ok, %{error: @no_uri_error}}
+  end
+
+  defp fetch_json(%{"tokenURI" => {:ok, ["http://" <> _ = token_uri]}}) do
+    fetch_metadata(token_uri)
+  end
+
+  defp fetch_json(%{"tokenURI" => {:ok, ["https://" <> _ = token_uri]}}) do
+    fetch_metadata(token_uri)
+  end
+
+  defp fetch_json(%{"tokenURI" => {:ok, [json]}}) do
+    Jason.decode(json)
+  rescue
+    e ->
+      Logger.error(fn -> ["Unknown metadata format #{inspect(json)}. error #{inspect(e)}"] end)
+
+      {:error, json}
   end
 
-  defp fetch_json(%{"tokenURI" => {:ok, [token_uri]}}) do
+  defp fetch_json(result) do
+    Logger.error(fn -> ["Unknown metadata format #{inspect(result)}."] end)
+
+    {:error, result}
+  end
+
+  defp fetch_metadata(token_uri) do
     case HTTPoison.get(token_uri) do
       {:ok, %Response{body: body, status_code: 200}} ->
-        Jason.decode(body)
+        {:ok, json} = Jason.decode(body)
+
+        {:ok, %{metadata: json}}
 
       {:ok, %Response{body: body}} ->
         {:error, body}
@@ -61,9 +92,10 @@ defmodule Explorer.Token.InstanceMetadataRetriever do
       {:error, %Error{reason: reason}} ->
         {:error, reason}
     end
-  end
+  rescue
+    e ->
+      Logger.error(fn -> ["Could not send request to token uri #{inspect(token_uri)}. error #{inspect(e)}"] end)
 
-  defp fetch_json(result) do
-    {:error, result}
+      {:error, :request_error}
   end
 end

apps/explorer/priv/repo/migrations/20191007082500_add_indexes_for_token_instances_query.exs

@@ -0,0 +1,8 @@
+defmodule Explorer.Repo.Migrations.AddIndexesForTokenInstrancesQuery do
+  use Ecto.Migration
+
+  def change do
+    create_if_not_exists(index(:token_instances, [:token_id]))
+    create_if_not_exists(index(:tokens, [:type]))
+  end
+end

apps/explorer/priv/repo/migrations/20191010075740_add_error_to_token_instances.exs

@@ -0,0 +1,11 @@
+defmodule Explorer.Repo.Migrations.AddErrorToTokenInstances do
+  use Ecto.Migration
+
+  def change do
+    alter table(:token_instances) do
+      add(:error, :string)
+    end
+
+    create(index(:token_instances, [:error]))
+  end
+end

apps/explorer/test/explorer/chain_test.exs

@@ -173,6 +173,42 @@ defmodule Explorer.ChainTest do
          valid?: false
        }} = Chain.upsert_token_instance(params)
     end
+
+    test "inserts just an error without metadata" do
+      token = insert(:token)
+      error = "no uri"
+
+      params = %{
+        token_id: 1,
+        token_contract_address_hash: token.contract_address_hash,
+        error: error
+      }
+
+      {:ok, result} = Chain.upsert_token_instance(params)
+
+      assert result.error == error
+    end
+
+    test "nillifies error" do
+      token = insert(:token)
+
+      insert(:token_instance,
+        token_id: 1,
+        token_contract_address_hash: token.contract_address_hash,
+        error: "no uri"
+      )
+
+      params = %{
+        token_id: 1,
+        token_contract_address_hash: token.contract_address_hash,
+        metadata: %{uri: "http://example1.com"}
+      }
+
+      {:ok, result} = Chain.upsert_token_instance(params)
+
+      assert is_nil(result.error)
+      assert result.metadata == params.metadata
+    end
   end
 
   describe "address_to_logs/2" do
@@ -3645,6 +3681,28 @@ defmodule Explorer.ChainTest do
       assert result.contract_address_hash == token_transfer.token_contract_address_hash
     end
 
+    test "does not fetch token transfers without token id" do
+      token_contract_address = insert(:contract_address)
+      token = insert(:token, contract_address: token_contract_address, type: "ERC-721")
+
+      transaction =
+        :transaction
+        |> insert()
+        |> with_block(insert(:block, number: 1))
+
+      insert(
+        :token_transfer,
+        block_number: 1000,
+        to_address: build(:address),
+        transaction: transaction,
+        token_contract_address: token_contract_address,
+        token: token,
+        token_id: nil
+      )
+
+      assert {:ok, []} = Chain.stream_unfetched_token_instances([], &[&1 | &2])
+    end
+
     test "do not fetch records with token instances" do
       token_contract_address = insert(:contract_address)
       token = insert(:token, contract_address: token_contract_address, type: "ERC-721")

apps/explorer/test/support/factory.ex

@@ -547,7 +547,8 @@ defmodule Explorer.Factory do
     %Instance{
       token_contract_address_hash: build(:address),
       token_id: 5,
-      metadata: %{key: "value"}
+      metadata: %{key: "value"},
+      error: nil
     }
   end
 

apps/indexer/config/config.exs

@@ -28,6 +28,12 @@ block_transformer =
       transformer
   end
 
+max_skipping_distance =
+  case Integer.parse(System.get_env("MAX_SKIPPING_DISTANCE", "")) do
+    {num, ""} -> num
+    _ -> 5
+  end
+
 config :indexer,
   block_transformer: block_transformer,
   ecto_repos: [Explorer.Repo],
@@ -36,7 +42,8 @@ config :indexer,
   # bytes
   memory_limit: 1 <<< 30,
   first_block: System.get_env("FIRST_BLOCK") || "0",
-  last_block: System.get_env("LAST_BLOCK") || ""
+  last_block: System.get_env("LAST_BLOCK") || "",
+  max_skipping_distance: max_skipping_distance
 
 # config :indexer, Indexer.Fetcher.ReplacedTransaction.Supervisor, disabled?: true
 # config :indexer, Indexer.Fetcher.BlockReward.Supervisor, disabled?: true

apps/indexer/lib/indexer/block/catchup/fetcher.ex

@@ -16,6 +16,7 @@ defmodule Indexer.Block.Catchup.Fetcher do
       async_import_replaced_transactions: 1,
       async_import_tokens: 1,
       async_import_token_balances: 1,
+      async_import_token_instances: 1,
       async_import_uncles: 1,
       fetch_and_import_range: 2
     ]
@@ -72,21 +73,12 @@ defmodule Indexer.Block.Catchup.Fetcher do
       ) do
     Logger.metadata(fetcher: :block_catchup)
 
-    {:ok, latest_block_number} =
-      case latest_block() do
-        nil ->
-          EthereumJSONRPC.fetch_block_number_by_tag("latest", json_rpc_named_arguments)
-
-        number ->
-          {:ok, number}
-      end
-
-    case latest_block_number do
+    case latest_block(json_rpc_named_arguments) do
       # let realtime indexer get the genesis block
       0 ->
         %{first_block_number: 0, missing_block_count: 0, shrunk: false}
 
-      _ ->
+      latest_block_number ->
         # realtime indexer gets the current latest block
         first = latest_block_number - 1
         last = last_block()
@@ -164,6 +156,7 @@ defmodule Indexer.Block.Catchup.Fetcher do
     async_import_token_balances(imported)
     async_import_uncles(imported)
     async_import_replaced_transactions(imported)
+    async_import_token_instances(imported)
   end
 
   defp stream_fetch_and_import(%__MODULE__{blocks_concurrency: blocks_concurrency} = state, sequence)
@@ -345,12 +338,23 @@ defmodule Indexer.Block.Catchup.Fetcher do
     end
   end
 
-  defp latest_block do
+  defp latest_block(json_rpc_named_arguments) do
     string_value = Application.get_env(:indexer, :last_block)
 
     case Integer.parse(string_value) do
-      {integer, ""} -> integer
-      _ -> nil
+      {integer, ""} ->
+        integer
+
+      _ ->
+        {:ok, number} = EthereumJSONRPC.fetch_block_number_by_tag("latest", json_rpc_named_arguments)
+        # leave to realtime indexer the blocks in the skipping window
+        skipping_distance = Application.get_env(:indexer, :max_skipping_distance)
+
+        if number > skipping_distance do
+          number - skipping_distance
+        else
+          0
+        end
     end
   end
 end

apps/indexer/lib/indexer/block/fetcher.ex

@@ -25,6 +25,7 @@ defmodule Indexer.Block.Fetcher do
     StakingPools,
     Token,
     TokenBalance,
+    TokenInstance,
     UncleBlock
   }
 
@@ -229,6 +230,12 @@ defmodule Indexer.Block.Fetcher do
     callback_module.import(state, options_with_broadcast)
   end
 
+  def async_import_token_instances(%{token_transfers: token_transfers}) do
+    TokenInstance.async_fetch(token_transfers)
+  end
+
+  def async_import_token_instances(_), do: :ok
+
   def async_import_block_rewards([]), do: :ok
 
   def async_import_block_rewards(errors) when is_list(errors) do

apps/indexer/lib/indexer/block/realtime/fetcher.ex

@@ -19,6 +19,7 @@ defmodule Indexer.Block.Realtime.Fetcher do
       async_import_replaced_transactions: 1,
       async_import_tokens: 1,
       async_import_token_balances: 1,
+      async_import_token_instances: 1,
       async_import_uncles: 1,
       fetch_and_import_range: 2,
       async_import_staking_pools: 0
@@ -87,9 +88,16 @@ defmodule Indexer.Block.Realtime.Fetcher do
     number = quantity_to_integer(quantity)
     # Subscriptions don't support getting all the blocks and transactions data,
     # so we need to go back and get the full block
-    start_fetch_and_import(number, block_fetcher, previous_number, max_number_seen)
+    {new_previous_number, new_max_number} =
+      case start_fetch_and_import(number, block_fetcher, previous_number, max_number_seen) do
+        # The number may have not been inserted if it was part of a small skip
+        :skip ->
+          Logger.debug(["#{inspect(number)} was skipped"])
+          {previous_number, max_number_seen}
 
-    new_max_number = new_max_number(number, max_number_seen)
+        _ ->
+          {number, new_max_number(number, max_number_seen)}
+      end
 
     Process.cancel_timer(timer)
     new_timer = schedule_polling()
@@ -97,7 +105,7 @@ defmodule Indexer.Block.Realtime.Fetcher do
     {:noreply,
      %{
        state
-       | previous_number: number,
+       | previous_number: new_previous_number,
          max_number_seen: new_max_number,
          timer: new_timer
      }}
@@ -115,7 +123,14 @@ defmodule Indexer.Block.Realtime.Fetcher do
     {number, new_max_number} =
       case EthereumJSONRPC.fetch_block_number_by_tag("latest", json_rpc_named_arguments) do
         {:ok, number} when is_nil(max_number_seen) or number > max_number_seen ->
-          start_fetch_and_import(number, block_fetcher, previous_number, number)
+          # in case of polling the realtime fetcher should take care of all the
+          # blocks in the skipping window, because the cathup fetcher wont
+          max_skipping_distance = Application.get_env(:indexer, :max_skipping_distance)
+
+          last_catchup_number = max(0, 10 - max_skipping_distance - 1)
+          starting_number = max(previous_number, last_catchup_number) || last_catchup_number
+
+          start_fetch_and_import(number, block_fetcher, starting_number, nil)
 
           {max_number_seen, number}
 
@@ -211,27 +226,31 @@ defmodule Indexer.Block.Realtime.Fetcher do
   end
 
   defp start_fetch_and_import(number, block_fetcher, previous_number, max_number_seen) do
-    start_at = determine_start_at(number, previous_number, max_number_seen)
+    fetching_action = determine_fetching_action(number, previous_number, max_number_seen)
 
-    for block_number_to_fetch <- start_at..number do
-      args = [block_number_to_fetch, block_fetcher, reorg?(number, max_number_seen)]
-      Task.Supervisor.start_child(TaskSupervisor, __MODULE__, :fetch_and_import_block, args)
+    if fetching_action != :skip do
+      for block_number_to_fetch <- fetching_action do
+        args = [block_number_to_fetch, block_fetcher, reorg?(number, max_number_seen)]
+        Task.Supervisor.start_child(TaskSupervisor, __MODULE__, :fetch_and_import_block, args)
+      end
     end
+
+    fetching_action
   end
 
-  defp determine_start_at(number, nil, nil), do: number
+  def determine_fetching_action(number, previous_number, max_number_seen) do
+    cond do
+      reorg?(number, max_number_seen) ->
+        [number]
 
-  defp determine_start_at(number, nil, max_number_seen) do
-    determine_start_at(number, number - 1, max_number_seen)
-  end
+      can_be_skipped?(number, max_number_seen) ->
+        :skip
 
-  defp determine_start_at(number, previous_number, max_number_seen) do
-    if reorg?(number, max_number_seen) do
-      # set start_at to NOT fill in skipped numbers
-      number
-    else
-      # set start_at to fill in skipped numbers, if any
-      previous_number + 1
+      is_nil(previous_number) ->
+        [number]
+
+      true ->
+        (previous_number + 1)..number
     end
   end
 
@@ -241,6 +260,14 @@ defmodule Indexer.Block.Realtime.Fetcher do
 
   defp reorg?(_, _), do: false
 
+  defp can_be_skipped?(number, max_number_seen) when is_integer(max_number_seen) and number > max_number_seen + 1 do
+    max_skipping_distance = Application.get_env(:indexer, :max_skipping_distance)
+
+    max_skipping_distance > 1 and number <= max_number_seen + max_skipping_distance
+  end
+
+  defp can_be_skipped?(_, _), do: false
+
   @reorg_delay 5_000
 
   @decorate trace(name: "fetch", resource: "Indexer.Block.Realtime.Fetcher.fetch_and_import_block/3", tracer: Tracer)
@@ -358,6 +385,7 @@ defmodule Indexer.Block.Realtime.Fetcher do
     async_import_internal_transactions(imported, Keyword.get(json_rpc_named_arguments, :variant))
     async_import_tokens(imported)
     async_import_token_balances(imported)
+    async_import_token_instances(imported)
     async_import_uncles(imported)
     async_import_replaced_transactions(imported)
     async_import_staking_pools()

apps/indexer/lib/indexer/fetcher/token_instance.ex

@@ -6,6 +6,8 @@ defmodule Indexer.Fetcher.TokenInstance do
   use Indexer.Fetcher
   use Spandex.Decorators
 
+  require Logger
+
   alias Explorer.Chain
   alias Explorer.Token.InstanceMetadataRetriever
   alias Indexer.BufferedTask
@@ -50,16 +52,35 @@ defmodule Indexer.Fetcher.TokenInstance do
   @impl BufferedTask
   def run([%{contract_address_hash: token_contract_address_hash, token_id: token_id}], _json_rpc_named_arguments) do
     case InstanceMetadataRetriever.fetch_metadata(to_string(token_contract_address_hash), Decimal.to_integer(token_id)) do
-      {:ok, metadata} ->
+      {:ok, %{metadata: metadata}} ->
+        params = %{
+          token_id: token_id,
+          token_contract_address_hash: token_contract_address_hash,
+          metadata: metadata,
+          error: nil
+        }
+
+        {:ok, _result} = Chain.upsert_token_instance(params)
+
+      {:ok, %{error: error}} ->
         params = %{
           token_id: token_id,
           token_contract_address_hash: token_contract_address_hash,
-          metadata: metadata
+          error: error
         }
 
         {:ok, _result} = Chain.upsert_token_instance(params)
 
-      _other ->
+      result ->
+        Logger.error(fn ->
+          [
+            "failed to fetch token instance metadata for #{
+              inspect({to_string(token_contract_address_hash), Decimal.to_integer(token_id)})
+            }: ",
+            inspect(result)
+          ]
+        end)
+
         :ok
     end
 
@@ -69,6 +90,18 @@ defmodule Indexer.Fetcher.TokenInstance do
   @doc """
   Fetches token instance data asynchronously.
   """
+  def async_fetch(token_transfers) when is_list(token_transfers) do
+    data =
+      token_transfers
+      |> Enum.reject(fn token_transfer -> is_nil(token_transfer.token_id) end)
+      |> Enum.map(fn token_transfer ->
+        %{contract_address_hash: token_transfer.token_contract_address_hash, token_id: token_transfer.token_id}
+      end)
+      |> Enum.uniq()
+
+    BufferedTask.buffer(__MODULE__, data)
+  end
+
   def async_fetch(data) do
     BufferedTask.buffer(__MODULE__, data)
   end

apps/indexer/test/indexer/block/catchup/bound_interval_supervisor_test.exs

@@ -28,6 +28,11 @@ defmodule Indexer.Block.Catchup.BoundIntervalSupervisorTest do
 
   setup :verify_on_exit!
 
+  # run the tests without the skipping window
+  setup do
+    Application.put_env(:indexer, :max_skipping_distance, 0)
+  end
+
   describe "start_link/1" do
     # See https://github.com/poanetwork/blockscout/issues/597
     @tag :no_geth

apps/indexer/test/indexer/block/catchup/fetcher_test.exs

@@ -32,6 +32,11 @@ defmodule Indexer.Block.Catchup.FetcherTest do
     }
   end
 
+  setup do
+    # run the tests without the skipping window
+    Application.put_env(:indexer, :max_skipping_distance, 0)
+  end
+
   describe "import/1" do
     test "fetches uncles asynchronously", %{json_rpc_named_arguments: json_rpc_named_arguments} do
       CoinBalance.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments)

apps/indexer/test/indexer/block/realtime/fetcher_test.exs

@@ -40,6 +40,11 @@ defmodule Indexer.Block.Realtime.FetcherTest do
     %{block_fetcher: block_fetcher, json_rpc_named_arguments: core_json_rpc_named_arguments}
   end
 
+  setup do
+    # run the tests with a realistic skipping window
+    Application.put_env(:indexer, :max_skipping_distance, 3)
+  end
+
   describe "Indexer.Block.Fetcher.fetch_and_import_range/1" do
     @tag :no_geth
     test "in range with internal transactions", %{
@@ -424,4 +429,51 @@ defmodule Indexer.Block.Realtime.FetcherTest do
               }} = Indexer.Block.Fetcher.fetch_and_import_range(block_fetcher, 3_946_079..3_946_080)
     end
   end
+
+  describe "determine_fetching_action/4" do
+    test "when everything (except number) is nil results in fetching only the number" do
+      assert [14] == Realtime.Fetcher.determine_fetching_action(14, nil, nil)
+    end
+
+    test "when number is also max_number_seen results in fetching only the number" do
+      number = 23
+      assert [number] == Realtime.Fetcher.determine_fetching_action(number, nil, number)
+      assert [number] == Realtime.Fetcher.determine_fetching_action(number, 21, number)
+    end
+
+    test "when max_number_seen is nil, fetching will start from previous_number" do
+      # note: this is a way to force this behavior, used by `poll_latest_block_number`
+      number = 156
+      previous_number = 150
+      old_number = 94
+
+      assert (previous_number + 1)..number == Realtime.Fetcher.determine_fetching_action(number, previous_number, nil)
+      assert (old_number + 1)..number == Realtime.Fetcher.determine_fetching_action(number, old_number, nil)
+    end
+
+    test "when number immediately follows the previous_number it is fetched" do
+      max_number_seen = 26
+      number = 27
+
+      assert [number] == Realtime.Fetcher.determine_fetching_action(number, nil, max_number_seen)
+    end
+
+    test "when number is inside the allowed skipping window nothing is fetched" do
+      max_number_seen = 26
+
+      assert :skip == Realtime.Fetcher.determine_fetching_action(28, nil, max_number_seen)
+      assert :skip == Realtime.Fetcher.determine_fetching_action(29, nil, max_number_seen)
+    end
+
+    test "when number is over the allowed skipping window all the values since the previous_number will be fetched" do
+      max_number_seen = 390
+      previous_number = 381
+      max_skipping_distance = Application.get_env(:indexer, :max_skipping_distance)
+
+      number = max_number_seen + max_skipping_distance + 1
+
+      assert (previous_number + 1)..number ==
+               Realtime.Fetcher.determine_fetching_action(number, previous_number, max_number_seen)
+    end
+  end
 end

docs/env-variables.md

@@ -70,3 +70,4 @@ $ export NETWORK=POA
 | `EMISSION_FORMAT` | | Should be set to `POA` if you have block emission indentical to POA Network. This env var is used only if `CHAIN_SPEC_PATH` is set | `STANDARD` | v2.0.4+ | | |
 | `REWARDS_CONTRACT_ADDRESS` | | Emission rewards contract address. This env var is used only if `EMISSION_FORMAT` is set to `POA` | `0xeca443e8e1ab29971a45a9c57a6a9875701698a5` | v2.0.4+ | | |
 | `BLOCKSCOUT_PROTOCOL` | | Url scheme for blockscout | in prod env `https` is used, in dev env `http` is used | master | | |
+| `MAX_SKIPPING_DISTANCE` | | The maximum distance the indexer is allowed to wait for when notified of a number not following the lask known one. | 4 | master | |

mix.lock

@@ -11,7 +11,7 @@
   "briefly": {:git, "https://github.com/CargoSense/briefly.git", "2526e9674a4e6996137e066a1295ea60962712b8", []},
   "bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], [], "hexpm"},
   "bypass": {:hex, :bypass, "1.0.0", "b78b3dcb832a71aca5259c1a704b2e14b55fd4e1327ff942598b4e7d1a7ad83d", [:mix], [{:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: false]}], "hexpm"},
-  "certifi": {:hex, :certifi, "2.3.1", "d0f424232390bf47d82da8478022301c561cf6445b5b5fb6a84d49a9e76d2639", [:rebar3], [{:parse_trans, "3.2.0", [hex: :parse_trans, repo: "hexpm", optional: false]}], "hexpm"},
+  "certifi": {:hex, :certifi, "2.5.1", "867ce347f7c7d78563450a18a6a28a8090331e77fa02380b4a21962a65d36ee5", [:rebar3], [{:parse_trans, "~>3.3", [hex: :parse_trans, repo: "hexpm", optional: false]}], "hexpm"},
   "cldr_utils": {:hex, :cldr_utils, "2.3.0", "e7e8b5ad7494a929c1b620cc489c3aa3f6e7e5299584c1a934bbdb56d1a53c70", [:mix], [{:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}], "hexpm"},
   "combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm"},
   "comeonin": {:hex, :comeonin, "4.1.2", "3eb5620fd8e35508991664b4c2b04dd41e52f1620b36957be837c1d7784b7592", [:mix], [{:argon2_elixir, "~> 1.2", [hex: :argon2_elixir, repo: "hexpm", optional: true]}, {:bcrypt_elixir, "~> 0.12.1 or ~> 1.0", [hex: :bcrypt_elixir, repo: "hexpm", optional: true]}, {:pbkdf2_elixir, "~> 0.12", [hex: :pbkdf2_elixir, repo: "hexpm", optional: true]}], "hexpm"},
@@ -53,10 +53,10 @@
   "flow": {:hex, :flow, "0.14.3", "0d92991fe53035894d24aa8dec10dcfccf0ae00c4ed436ace3efa9813a646902", [:mix], [{:gen_stage, "~> 0.14.0", [hex: :gen_stage, repo: "hexpm", optional: false]}], "hexpm"},
   "gen_stage": {:hex, :gen_stage, "0.14.2", "6a2a578a510c5bfca8a45e6b27552f613b41cf584b58210f017088d3d17d0b14", [:mix], [], "hexpm"},
   "gettext": {:hex, :gettext, "0.16.1", "e2130b25eebcbe02bb343b119a07ae2c7e28bd4b146c4a154da2ffb2b3507af2", [:mix], [], "hexpm"},
-  "hackney": {:hex, :hackney, "1.13.0", "24edc8cd2b28e1c652593833862435c80661834f6c9344e84b6a2255e7aeef03", [:rebar3], [{:certifi, "2.3.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "5.1.2", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "1.0.2", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"},
+  "hackney": {:hex, :hackney, "1.15.2", "07e33c794f8f8964ee86cebec1a8ed88db5070e52e904b8f12209773c1036085", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.5", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"},
   "html_entities": {:hex, :html_entities, "0.4.0", "f2fee876858cf6aaa9db608820a3209e45a087c5177332799592142b50e89a6b", [:mix], [], "hexpm"},
   "httpoison": {:hex, :httpoison, "1.0.0", "1f02f827148d945d40b24f0b0a89afe40bfe037171a6cf70f2486976d86921cd", [:mix], [{:hackney, "~> 1.8", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"},
-  "idna": {:hex, :idna, "5.1.2", "e21cb58a09f0228a9e0b95eaa1217f1bcfc31a1aaa6e1fdf2f53a33f7dbd9494", [:rebar3], [{:unicode_util_compat, "0.3.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm"},
+  "idna": {:hex, :idna, "6.0.0", "689c46cbcdf3524c44d5f3dde8001f364cd7608a99556d8fbd8239a5798d4c10", [:rebar3], [{:unicode_util_compat, "0.4.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm"},
   "jason": {:hex, :jason, "1.1.2", "b03dedea67a99223a2eaf9f1264ce37154564de899fd3d8b9a21b1a6fd64afe7", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm"},
   "jsx": {:hex, :jsx, "2.8.3", "a05252d381885240744d955fbe3cf810504eb2567164824e19303ea59eef62cf", [:mix, :rebar3], [], "hexpm"},
   "junit_formatter": {:hex, :junit_formatter, "3.0.1", "4ed76a50886717a6d683a978cec775abdcb88d9d51cfddd3d8fbf8e6af4625da", [:mix], [], "hexpm"},
@@ -70,7 +70,7 @@
   "memento": {:hex, :memento, "0.3.1", "b2909390820550d8b90b68ec96f9e15ff8a45a28b6f97fa4a62ef50e87c2f9d9", [:mix], [], "hexpm"},
   "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm"},
   "mime": {:hex, :mime, "1.3.1", "30ce04ab3175b6ad0bdce0035cba77bba68b813d523d1aac73d9781b4d193cf8", [:mix], [], "hexpm"},
-  "mimerl": {:hex, :mimerl, "1.0.2", "993f9b0e084083405ed8252b99460c4f0563e41729ab42d9074fd5e52439be88", [:rebar3], [], "hexpm"},
+  "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm"},
   "mix_erlang_tasks": {:hex, :mix_erlang_tasks, "0.1.0", "36819fec60b80689eb1380938675af215565a89320a9e29c72c70d97512e4649", [:mix], [], "hexpm"},
   "mochiweb": {:hex, :mochiweb, "2.18.0", "eb55f1db3e6e960fac4e6db4e2db9ec3602cc9f30b86cd1481d56545c3145d2e", [:rebar3], [], "hexpm"},
   "mock": {:hex, :mock, "0.3.3", "42a433794b1291a9cf1525c6d26b38e039e0d3a360732b5e467bfc77ef26c914", [:mix], [{:meck, "~> 0.8.13", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm"},
@@ -80,7 +80,7 @@
   "nimble_parsec": {:hex, :nimble_parsec, "0.5.1", "c90796ecee0289dbb5ad16d3ad06f957b0cd1199769641c961cfe0b97db190e0", [:mix], [], "hexpm"},
   "optimal": {:hex, :optimal, "0.3.6", "46bbf52fbbbd238cda81e02560caa84f93a53c75620f1fe19e81e4ae7b07d1dd", [:mix], [], "hexpm"},
   "parallel_stream": {:hex, :parallel_stream, "1.0.6", "b967be2b23f0f6787fab7ed681b4c45a215a81481fb62b01a5b750fa8f30f76c", [:mix], [], "hexpm"},
-  "parse_trans": {:hex, :parse_trans, "3.2.0", "2adfa4daf80c14dc36f522cf190eb5c4ee3e28008fc6394397c16f62a26258c2", [:rebar3], [], "hexpm"},
+  "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm"},
   "phoenix": {:hex, :phoenix, "1.4.0", "56fe9a809e0e735f3e3b9b31c1b749d4b436e466d8da627b8d82f90eaae714d2", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"},
   "phoenix_ecto": {:hex, :phoenix_ecto, "4.0.0", "c43117a136e7399ea04ecaac73f8f23ee0ffe3e07acfcb8062fe5f4c9f0f6531", [:mix], [{:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.9", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"},
   "phoenix_form_awesomplete": {:hex, :phoenix_form_awesomplete, "0.1.5", "d09aade160b584e3428e1e095645482396f17bddda4f566f1118f12d2598d11c", [:mix], [{:phoenix_html, "~> 2.10", [hex: :phoenix_html, repo: "hexpm", optional: false]}], "hexpm"},
@@ -107,11 +107,11 @@
   "spandex_datadog": {:hex, :spandex_datadog, "0.4.0", "75113a73e843123074886a2e31994af07d6e0632749a8d97e9ca6157b120c287", [:mix], [{:msgpax, "~> 2.2.1", [hex: :msgpax, repo: "hexpm", optional: false]}, {:spandex, "~> 2.3", [hex: :spandex, repo: "hexpm", optional: false]}], "hexpm"},
   "spandex_ecto": {:hex, :spandex_ecto, "0.4.0", "deaeaddc11a35f1c551206c53d09bb93a0da5808dbef751430e465c8c7de01d3", [:mix], [{:spandex, "~> 2.2", [hex: :spandex, repo: "hexpm", optional: false]}], "hexpm"},
   "spandex_phoenix": {:hex, :spandex_phoenix, "0.3.2", "e81889d80852a895cf62ce2e25181b15766d21e8647962e0a4458414b935feb3", [:mix], [{:plug, "~> 1.3", [hex: :plug, repo: "hexpm", optional: false]}, {:spandex, "~> 2.2", [hex: :spandex, repo: "hexpm", optional: false]}], "hexpm"},
-  "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.1", "28a4d65b7f59893bc2c7de786dec1e1555bd742d336043fe644ae956c3497fbe", [:make, :rebar], [], "hexpm"},
+  "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.5", "6eaf7ad16cb568bb01753dbbd7a95ff8b91c7979482b95f38443fe2c8852a79b", [:make, :mix, :rebar3], [], "hexpm"},
   "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"},
   "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"},
   "tzdata": {:hex, :tzdata, "1.0.1", "f6027a331af7d837471248e62733c6ebee86a72e57c613aa071ebb1f750fc71a", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"},
-  "unicode_util_compat": {:hex, :unicode_util_compat, "0.3.1", "a1f612a7b512638634a603c8f401892afbf99b8ce93a45041f8aaca99cadb85e", [:rebar3], [], "hexpm"},
+  "unicode_util_compat": {:hex, :unicode_util_compat, "0.4.1", "d869e4c68901dd9531385bb0c8c40444ebf624e60b6962d95952775cac5e90cd", [:rebar3], [], "hexpm"},
   "wallaby": {:hex, :wallaby, "0.22.0", "e5d16bfa7ab23562c8a6e3b0a31445a2fd470ca622082a910114807ba823780d", [:mix], [{:httpoison, "~> 0.12 or ~> 1.0", [hex: :httpoison, repo: "hexpm", optional: false]}, {:poison, ">= 1.4.0", [hex: :poison, repo: "hexpm", optional: false]}, {:poolboy, "~> 1.5", [hex: :poolboy, repo: "hexpm", optional: false]}], "hexpm"},
   "websocket_client": {:hex, :websocket_client, "1.3.0", "2275d7daaa1cdacebf2068891c9844b15f4fdc3de3ec2602420c2fb486db59b6", [:rebar3], [], "hexpm"},
   "wobserver": {:git, "https://github.com/poanetwork/wobserver.git", "13bcda30a87f4f0be1878920a79433ad831eefbe", [branch: "support-https"]},