From 1223faae6d4287d567bd7eb621744a35f03112df Mon Sep 17 00:00:00 2001 From: nikitosing <32202610+nikitosing@users.noreply.github.com> Date: Tue, 5 Nov 2024 14:58:58 +0300 Subject: [PATCH] fix: Fix sitemap timeout; optimize OrderedCache preloads (#11131) * fix: Fix sitemap timeout; optimize OrderedCache preloads * Process review comments --- .../lib/block_scout_web/templates/robots/sitemap.xml.eex | 4 ++-- .../lib/block_scout_web/views/robots_view.ex | 2 +- apps/explorer/lib/explorer/chain.ex | 2 +- apps/explorer/lib/explorer/chain/address.ex | 9 ++++++++- apps/explorer/lib/explorer/chain/ordered_cache.ex | 3 ++- apps/explorer/lib/explorer/chain/token.ex | 9 +++++++-- 6 files changed, 21 insertions(+), 8 deletions(-) diff --git a/apps/block_scout_web/lib/block_scout_web/templates/robots/sitemap.xml.eex b/apps/block_scout_web/lib/block_scout_web/templates/robots/sitemap.xml.eex index 0e67d48ab1..e633bc29e3 100644 --- a/apps/block_scout_web/lib/block_scout_web/templates/robots/sitemap.xml.eex +++ b/apps/block_scout_web/lib/block_scout_web/templates/robots/sitemap.xml.eex @@ -2,7 +2,7 @@ <% host = APIDocsView.blockscout_url(true) %> <% date = to_string(Date.utc_today()) %> <% non_parameterized_urls = ["/", "/txs", "/blocks", "/accounts", "/verified-contracts", "/tokens", "/apps", "/stats", "/api-docs", "/graphiql", "/search-results", "/withdrawals", "/l2-deposits", "/l2-output-roots", "/l2-txn-batches", "/l2-withdrawals"] %> -<% params = [paging_options: %PagingOptions{page_size: limit()}] %> +<% params = [paging_options: %PagingOptions{page_size: limit()}, necessity_by_association: %{}] %> <%= for url <- non_parameterized_urls do %> @@ -50,4 +50,4 @@ <%= date %> <% end %> - \ No newline at end of file + diff --git a/apps/block_scout_web/lib/block_scout_web/views/robots_view.ex b/apps/block_scout_web/lib/block_scout_web/views/robots_view.ex index 9939ec3e68..628ed672e0 100644 --- a/apps/block_scout_web/lib/block_scout_web/views/robots_view.ex +++ b/apps/block_scout_web/lib/block_scout_web/views/robots_view.ex @@ -5,6 +5,6 @@ defmodule BlockScoutWeb.RobotsView do alias Explorer.{Chain, PagingOptions} alias Explorer.Chain.{Address, Token} - @limit 200 + @limit 50 defp limit, do: @limit end diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index cda1bec0ee..3065e3ada2 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -795,7 +795,7 @@ defmodule Explorer.Chain do from(contract in SmartContract, inner_join: address in Address, on: contract.address_hash == address.hash, - order_by: [desc: address.transactions_count], + order_by: [desc: address.fetched_coin_balance], limit: ^limit, select: contract.address_hash ) diff --git a/apps/explorer/lib/explorer/chain/address.ex b/apps/explorer/lib/explorer/chain/address.ex index da1ea6eda5..0729633b38 100644 --- a/apps/explorer/lib/explorer/chain/address.ex +++ b/apps/explorer/lib/explorer/chain/address.ex @@ -543,6 +543,13 @@ defmodule Explorer.Chain.Address do defp fetch_top_addresses(options) do paging_options = Keyword.get(options, :paging_options, @default_paging_options) + necessity_by_association = + Keyword.get(options, :necessity_by_association, %{ + :names => :optional, + :smart_contract => :optional, + proxy_implementations_association() => :optional + }) + case paging_options do %PagingOptions{key: {0, _hash}} -> [] @@ -552,11 +559,11 @@ defmodule Explorer.Chain.Address do from(a in Address, where: a.fetched_coin_balance > ^0, order_by: [desc: a.fetched_coin_balance, asc: a.hash], - preload: [:names, :smart_contract, ^proxy_implementations_association()], select: {a, a.transactions_count} ) base_query + |> Chain.join_associations(necessity_by_association) |> ExplorerHelper.maybe_hide_scam_addresses(:hash) |> page_addresses(paging_options) |> limit(^paging_options.page_size) diff --git a/apps/explorer/lib/explorer/chain/ordered_cache.ex b/apps/explorer/lib/explorer/chain/ordered_cache.ex index 46b1f8f939..631bbf8ab7 100644 --- a/apps/explorer/lib/explorer/chain/ordered_cache.ex +++ b/apps/explorer/lib/explorer/chain/ordered_cache.ex @@ -247,9 +247,10 @@ defmodule Explorer.Chain.OrderedCache do ConCache.update(cache_name(), ids_list_key(), fn ids -> updated_list = elements + |> Enum.sort_by(&element_to_id(&1), &prevails?(&1, &2)) + |> Enum.take(max_size()) |> do_preloads() |> Enum.map(&{element_to_id(&1), &1}) - |> Enum.sort(&prevails?(&1, &2)) |> merge_and_update(ids || [], max_size()) # ids_list is set to never expire diff --git a/apps/explorer/lib/explorer/chain/token.ex b/apps/explorer/lib/explorer/chain/token.ex index f97e0f5113..83f1bd9d30 100644 --- a/apps/explorer/lib/explorer/chain/token.ex +++ b/apps/explorer/lib/explorer/chain/token.ex @@ -201,16 +201,21 @@ defmodule Explorer.Chain.Token do Chain.paging_options() | {:sorting, SortingHelper.sorting_params()} | {:token_type, [String.t()]} + | {:necessity_by_association, map()} ]) :: [Token.t()] def list_top(filter, options \\ []) do paging_options = Keyword.get(options, :paging_options, Chain.default_paging_options()) token_type = Keyword.get(options, :token_type, nil) sorting = Keyword.get(options, :sorting, []) - query = from(t in Token, preload: [:contract_address]) + necessity_by_association = + Keyword.get(options, :necessity_by_association, %{ + :contract_address => :optional + }) sorted_paginated_query = - query + Token + |> Chain.join_associations(necessity_by_association) |> ExplorerHelper.maybe_hide_scam_addresses(:contract_address_hash) |> apply_filter(token_type) |> SortingHelper.apply_sorting(sorting, @default_sorting)