check cache status in chain module

apps/block_scout_web/lib/block_scout_web/controllers/api/v1/health_controller.ex

@@ -1,31 +1,25 @@
 defmodule BlockScoutWeb.API.V1.HealthController do
   use BlockScoutWeb, :controller
 
-  alias Explorer.{Chain, PagingOptions}
+  alias Explorer.Chain
 
   def health(conn, _) do
-    with {:ok, number, timestamp} <- Chain.last_block_status() do
-      send_resp(conn, :ok, result(number, timestamp))
+    with {:ok, number, timestamp} <- Chain.last_db_block_status(),
+         {:ok, cache_number, cache_timestamp} <- Chain.last_cache_block_status() do
+      send_resp(conn, :ok, result(number, timestamp, cache_number, cache_timestamp))
     else
       status -> send_resp(conn, :internal_server_error, error(status))
     end
   end
 
-  def result(number, timestamp) do
-    latest_block_in_cache =
-      [
-        paging_options: %PagingOptions{page_size: 1}
-      ]
-      |> Chain.list_blocks()
-      |> List.last()
-
+  def result(number, timestamp, cache_number, cache_timestamp) do
     %{
       "healthy" => true,
       "data" => %{
-        "db_latest_block_number" => to_string(number),
-        "db_latest_block_inserted_at" => to_string(timestamp),
-        "cache_latest_block_number" => to_string(latest_block_in_cache.number),
-        "cache_latest_block_inserted_at" => to_string(latest_block_in_cache.timestamp)
+        "latest_block_number" => to_string(number),
+        "latest_block_inserted_at" => to_string(timestamp),
+        "cache_latest_block_number" => to_string(cache_number),
+        "cache_latest_block_inserted_at" => to_string(cache_timestamp)
       }
     }
     |> Jason.encode!()
@@ -49,8 +43,8 @@ defmodule BlockScoutWeb.API.V1.HealthController do
       "error_description" =>
         "There are no new blocks in the DB for the last 5 mins. Check the healthiness of Ethereum archive node or the Blockscout DB instance",
       "data" => %{
-        "db_latest_block_number" => to_string(number),
-        "db_latest_block_inserted_at" => to_string(timestamp)
+        "latest_block_number" => to_string(number),
+        "latest_block_inserted_at" => to_string(timestamp)
       }
     }
     |> Jason.encode!()

apps/block_scout_web/test/block_scout_web/controllers/api/v1/health_controller_test.exs

@@ -1,6 +1,15 @@
 defmodule BlockScoutWeb.API.V1.HealthControllerTest do
   use BlockScoutWeb.ConnCase
 
+  alias Explorer.{Chain, PagingOptions}
+
+  setup do
+    Supervisor.terminate_child(Explorer.Supervisor, {ConCache, Explorer.Chain.Cache.Blocks.cache_name()})
+    Supervisor.restart_child(Explorer.Supervisor, {ConCache, Explorer.Chain.Cache.Blocks.cache_name()})
+
+    :ok
+  end
+
   describe "GET last_block_status/0" do
     test "returns error when there are no blocks in db", %{conn: conn} do
       request = get(conn, api_v1_health_path(conn, :health))
@@ -25,8 +34,8 @@ defmodule BlockScoutWeb.API.V1.HealthControllerTest do
                "error_description" =>
                  "There are no new blocks in the DB for the last 5 mins. Check the healthiness of Ethereum archive node or the Blockscout DB instance",
                "data" => %{
-                 "db_latest_block_number" => _,
-                 "db_latest_block_inserted_at" => _
+                 "latest_block_number" => _,
+                 "latest_block_inserted_at" => _
                }
              } = Poison.decode!(request.resp_body)
     end
@@ -44,11 +53,38 @@ defmodule BlockScoutWeb.API.V1.HealthControllerTest do
       assert result["healthy"] == true
 
       assert %{
-               "db_latest_block_number" => to_string(block1.number),
-               "db_latest_block_inserted_at" => to_string(block1.timestamp),
+               "latest_block_number" => to_string(block1.number),
+               "latest_block_inserted_at" => to_string(block1.timestamp),
                "cache_latest_block_number" => to_string(block1.number),
                "cache_latest_block_inserted_at" => to_string(block1.timestamp)
              } == result["data"]
     end
   end
+
+  test "return error when cache is stale", %{conn: conn} do
+    stale_block = insert(:block, consensus: true, timestamp: Timex.shift(DateTime.utc_now(), hours: -50), number: 3)
+    state_block_hash = stale_block.hash
+
+    assert [%{hash: ^state_block_hash}] = Chain.list_blocks(paging_options: %PagingOptions{page_size: 1})
+
+    insert(:block, consensus: true, timestamp: DateTime.utc_now(), number: 1)
+
+    assert [%{hash: ^state_block_hash}] = Chain.list_blocks(paging_options: %PagingOptions{page_size: 1})
+
+    request = get(conn, api_v1_health_path(conn, :health))
+
+    assert request.status == 500
+
+    assert %{
+             "healthy" => false,
+             "error_code" => 5001,
+             "error_title" => "blocks fetching is stuck",
+             "error_description" =>
+               "There are no new blocks in the DB for the last 5 mins. Check the healthiness of Ethereum archive node or the Blockscout DB instance",
+             "data" => %{
+               "latest_block_number" => _,
+               "latest_block_inserted_at" => _
+             }
+           } = Poison.decode!(request.resp_body)
+  end
 end

apps/explorer/lib/explorer/chain.ex

@@ -1818,7 +1818,7 @@ defmodule Explorer.Chain do
     Repo.one!(query)
   end
 
-  def last_block_status do
+  def last_db_block_status do
     query =
       from(block in Block,
         select: {block.number, block.timestamp},
@@ -1827,11 +1827,27 @@ defmodule Explorer.Chain do
         limit: 1
       )
 
-    case Repo.one(query) do
-      nil ->
-        {:error, :no_blocks}
+    query
+    |> Repo.one()
+    |> block_status()
+  end
+
+  def last_cache_block_status do
+    [
+      paging_options: %PagingOptions{page_size: 1}
+    ]
+    |> list_blocks()
+    |> List.last()
+    |> case do
+      %{timestamp: timestamp, number: number} ->
+        block_status({number, timestamp})
 
-      {number, timestamp} ->
+      _ ->
+        block_status(nil)
+    end
+  end
+
+  defp block_status({number, timestamp}) do
     now = DateTime.utc_now()
     last_block_period = DateTime.diff(now, timestamp, :millisecond)
 
@@ -1841,7 +1857,8 @@ defmodule Explorer.Chain do
       {:ok, number, timestamp}
     end
   end
-  end
+
+  defp block_status(nil), do: {:error, :no_blocks}
 
   @doc """
   Calculates the ranges of missing consensus blocks in `range`.

apps/explorer/test/explorer/chain_test.exs

@@ -50,21 +50,35 @@ defmodule Explorer.ChainTest do
     end
   end
 
-  describe "last_block_status/0" do
+  describe "last_db_block_status/0" do
     test "return no_blocks errors if db is empty" do
-      assert {:error, :no_blocks} = Chain.last_block_status()
+      assert {:error, :no_blocks} = Chain.last_db_block_status()
     end
 
     test "returns {:ok, last_block_period} if block is in healthy period" do
       insert(:block, consensus: true)
 
-      assert {:ok, _, _} = Chain.last_block_status()
+      assert {:ok, _, _} = Chain.last_db_block_status()
     end
 
     test "return {:ok, last_block_period} if block is not in healthy period" do
       insert(:block, consensus: true, timestamp: Timex.shift(DateTime.utc_now(), hours: -50))
 
-      assert {:error, _, _} = Chain.last_block_status()
+      assert {:error, _, _} = Chain.last_db_block_status()
+    end
+  end
+
+  describe "last_cache_block_status/0" do
+    test "returns success if cache is not stale" do
+      insert(:block, consensus: true)
+
+      assert {:ok, _, _} = Chain.last_cache_block_status()
+    end
+
+    test "return error if cache is stale" do
+      insert(:block, consensus: true, timestamp: Timex.shift(DateTime.utc_now(), hours: -50))
+
+      assert {:error, _, _} = Chain.last_cache_block_status()
     end
   end