Add sitemap.xml

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### Features
 
+- [#8382](https://github.com/blockscout/blockscout/pull/8382) - Add sitemap.xml
 - [#8313](https://github.com/blockscout/blockscout/pull/8313) - Add batches to TokenInstance fetchers
 - [#8285](https://github.com/blockscout/blockscout/pull/8285) - Add CG/CMC coin price sources
 - [#8181](https://github.com/blockscout/blockscout/pull/8181) - Insert current token balances placeholders along with historical

apps/block_scout_web/lib/block_scout_web/controllers/robots_controller.ex

@@ -0,0 +1,13 @@
+defmodule BlockScoutWeb.RobotsController do
+  use BlockScoutWeb, :controller
+
+  def robots(conn, _params) do
+    conn
+    |> render("robots.txt")
+  end
+
+  def sitemap(conn, _params) do
+    conn
+    |> render("sitemap.xml")
+  end
+end

apps/block_scout_web/lib/block_scout_web/endpoint.ex

@@ -29,7 +29,6 @@ defmodule BlockScoutWeb.Endpoint do
       browserconfig.xml
       mstile-150x150.png
       safari-pinned-tab.svg
-      robots.txt
     ),
     only_matching: ~w(manifest)
   )

apps/block_scout_web/lib/block_scout_web/router.ex

@@ -47,6 +47,8 @@ defmodule BlockScoutWeb.Router do
   scope "/", BlockScoutWeb do
     pipe_through(:browser)
 
+    get("/robots.txt", RobotsController, :robots)
+    get("/sitemap.xml", RobotsController, :sitemap)
     get("/api-docs", APIDocsController, :index)
     get("/eth-rpc-api-docs", APIDocsController, :eth_rpc)
   end

apps/block_scout_web/lib/block_scout_web/templates/robots/robots.txt.eex

@@ -3,3 +3,4 @@
 # To ban all spiders from the entire site uncomment the next two lines:
 # User-agent: *
 # Disallow: /
+Sitemap: <%= APIDocsView.blockscout_url(true) %>/sitemap.xml

apps/block_scout_web/lib/block_scout_web/templates/robots/sitemap.xml.eex

@@ -0,0 +1,53 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<% host = APIDocsView.blockscout_url(true) %>
+<% date = to_string(Date.utc_today()) %>
+<% non_parameterized_urls = ["/", "/txs", "/blocks", "/accounts", "/verified-contracts", "/tokens", "/apps", "/stats", "/api-docs", "/graphiql", "/search-results", "/withdrawals", "/l2-deposits", "/l2-output-roots", "/l2-txn-batches", "/l2-withdrawals"] %>
+<% params = [paging_options: %PagingOptions{page_size: limit()}] %>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <%= for url <- non_parameterized_urls do %>
+    <url>
+      <loc><%= host %><%= url %></loc>
+      <lastmod><%= date %></lastmod>
+    </url>
+  <% end %>
+
+  <% addresses = Chain.list_top_addresses(params) %>
+  <%= for {address, _} <- addresses do %>
+    <url>
+      <loc><%= host %>/address/<%= to_string(address) %></loc>
+      <lastmod><%= date %></lastmod>
+    </url>
+  <% end %>
+
+  <% txs = Chain.recent_transactions(params, [:validated]) %>
+  <%= for tx <- txs do %>
+    <url>
+      <loc><%= host %>/tx/<%= to_string(tx.hash) %></loc>
+      <lastmod><%= date %></lastmod>
+    </url>
+  <% end %>
+
+  <% blocks = Chain.list_blocks(params) %>
+  <%= for block <- blocks do %>
+    <url>
+      <loc><%= host %>/block/<%= to_string(block.number) %></loc>
+      <lastmod><%= date %></lastmod>
+    </url>
+  <% end %>
+
+  <% tokens = Chain.list_top_tokens(nil, params) %>
+  <%= for token <- tokens do %>
+    <url>
+      <loc><%= host %>/token/<%= to_string(token.contract_address_hash) %></loc>
+      <lastmod><%= date %></lastmod>
+    </url>
+  <% end %>
+
+  <% smart_contracts_hashes = Chain.verified_contracts_top(limit()) %>
+  <%= for hash <- smart_contracts_hashes do %>
+    <url>
+      <loc><%= host %>/address/<%= Address.checksum(hash) %>?tab=contract</loc>
+      <lastmod><%= date %></lastmod>
+    </url>
+  <% end %>
+</urlset>

apps/block_scout_web/lib/block_scout_web/views/robots_view.ex

@@ -0,0 +1,10 @@
+defmodule BlockScoutWeb.RobotsView do
+  use BlockScoutWeb, :view
+
+  alias BlockScoutWeb.APIDocsView
+  alias Explorer.{Chain, PagingOptions}
+  alias Explorer.Chain.Address
+
+  @limit 200
+  defp limit, do: @limit
+end

apps/explorer/lib/explorer/chain.ex

@@ -6321,4 +6321,18 @@ defmodule Explorer.Chain do
       }
     )
   end
+
+  @spec verified_contracts_top(non_neg_integer()) :: [Hash.Address.t()]
+  def verified_contracts_top(limit) do
+    query =
+      from(contract in SmartContract,
+        inner_join: address in Address,
+        on: contract.address_hash == address.hash,
+        order_by: [desc: address.transactions_count],
+        limit: ^limit,
+        select: contract.address_hash
+      )
+
+    Repo.all(query)
+  end
 end

cspell.json

@@ -520,7 +520,9 @@
         "Asfpp",
         "Nerg",
         "secp",
-        "qwertyuioiuytrewertyuioiuytrertyuio"
+        "qwertyuioiuytrewertyuioiuytrertyuio",
+        "urlset",
+        "lastmod"
     ],
     "enableFiletypes": [
         "dotenv",