Add caches updating independently from Indexer

CHANGELOG.md

@@ -10,6 +10,7 @@
 - [#2497](https://github.com/poanetwork/blockscout/pull/2497) - Add generic Ordered Cache behaviour and implementation
 
 ### Fixes
+- [#2612](https://github.com/poanetwork/blockscout/pull/2612) - Add cache updating independently from Indexer
 - [#2659](https://github.com/poanetwork/blockscout/pull/2659) - Multipurpose front-end part update
 - [#2468](https://github.com/poanetwork/blockscout/pull/2468) - fix confirmations for non consensus blocks
 - [#2610](https://github.com/poanetwork/blockscout/pull/2610) - use CoinGecko instead of CoinMarketcap for exchange rates

apps/explorer/config/config.exs

@@ -29,7 +29,10 @@ config :explorer, Explorer.Counters.AverageBlockTime,
 
 config :explorer, Explorer.ChainSpec.GenesisData, enabled: false, chain_spec_path: System.get_env("CHAIN_SPEC_PATH")
 
-config :explorer, Explorer.Chain.Cache.BlockNumber, enabled: true
+config :explorer, Explorer.Chain.Cache.BlockNumber,
+  enabled: true,
+  ttl_check_interval: if(System.get_env("DISABLE_INDEXER") == "true", do: :timer.seconds(1), else: false),
+  global_ttl: if(System.get_env("DISABLE_INDEXER") == "true", do: :timer.seconds(5))
 
 config :explorer, Explorer.ExchangeRates.Source.CoinGecko, coin_id: System.get_env("COIN_GECKO_ID", "poa-network")
 
@@ -123,6 +126,14 @@ market_history_cache_period =
 
 config :explorer, Explorer.Market.MarketHistoryCache, period: market_history_cache_period
 
+config :explorer, Explorer.Chain.Cache.Blocks,
+  ttl_check_interval: if(System.get_env("DISABLE_INDEXER") == "true", do: :timer.seconds(1), else: false),
+  global_ttl: if(System.get_env("DISABLE_INDEXER") == "true", do: :timer.seconds(5))
+
+config :explorer, Explorer.Chain.Cache.Transactions,
+  ttl_check_interval: if(System.get_env("DISABLE_INDEXER") == "true", do: :timer.seconds(1), else: false),
+  global_ttl: if(System.get_env("DISABLE_INDEXER") == "true", do: :timer.seconds(5))
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"

apps/explorer/lib/explorer/application.ex

@@ -56,9 +56,7 @@ defmodule Explorer.Application do
 
     opts = [strategy: :one_for_one, name: Explorer.Supervisor]
 
-    res = Supervisor.start_link(children, opts)
+    Supervisor.start_link(children, opts)
-
-    res
   end
 
   defp configurable_children do

apps/explorer/lib/explorer/chain/cache/block_number.ex

@@ -7,7 +7,9 @@ defmodule Explorer.Chain.Cache.BlockNumber do
 
   use Explorer.Chain.MapCache,
     name: :block_number,
-    keys: [:min, :max]
+    keys: [:min, :max],
+    ttl_check_interval: Application.get_env(:explorer, __MODULE__)[:ttl_check_interval],
+    global_ttl: Application.get_env(:explorer, __MODULE__)[:global_ttl]
 
   alias Explorer.Chain
 

apps/explorer/lib/explorer/chain/cache/blocks.ex

@@ -11,7 +11,9 @@ defmodule Explorer.Chain.Cache.Blocks do
     ids_list_key: "block_numbers",
     preload: :transactions,
     preload: [miner: :names],
-    preload: :rewards
+    preload: :rewards,
+    ttl_check_interval: Application.get_env(:explorer, __MODULE__)[:ttl_check_interval],
+    global_ttl: Application.get_env(:explorer, __MODULE__)[:global_ttl]
 
   @type element :: Block.t()
 

apps/explorer/lib/explorer/chain/cache/transactions.ex

@@ -16,7 +16,9 @@ defmodule Explorer.Chain.Cache.Transactions do
       token_transfers: :token,
       token_transfers: :from_address,
       token_transfers: :to_address
-    ]
+    ],
+    ttl_check_interval: Application.get_env(:explorer, __MODULE__)[:ttl_check_interval],
+    global_ttl: Application.get_env(:explorer, __MODULE__)[:global_ttl]
 
   @type element :: Transaction.t()
 

apps/explorer/lib/explorer/chain/ordered_cache.ex

@@ -26,7 +26,10 @@ defmodule Explorer.Chain.OrderedCache do
       preload: [transaction: :hash]
   ```
   Additionally all of the options accepted by `ConCache.start_link/1` can be
-  provided as well. By default only `ttl_check_interval:` is set (to `false`).
+  provided as well. Unless specified, only these values have defaults:
+  - `:ttl_check_interval` is set (to `false`).
+  - `:callback` is only set if `:ttl_check_interval` is not `false` to call the
+    `remove_deleted_from_index` function, that removes expired values from the index.
 
   It's also possible, and advised, to override the implementation of the `c:prevails?/2`
   and `c:element_to_id/1` callbacks.
@@ -131,10 +134,7 @@ defmodule Explorer.Chain.OrderedCache do
     max_size = Keyword.get(opts, :max_size, 100)
     preloads = Keyword.get(opts, :preloads) || Keyword.get_values(opts, :preload)
 
-    concache_params =
+    concache_params = Keyword.drop(opts, [:ids_list_key, :max_size, :preloads, :preload])
-      opts
-      |> Keyword.drop([:ids_list_key, :max_size, :preloads, :preload])
-      |> Keyword.put_new(:ttl_check_interval, false)
 
     # credo:disable-for-next-line Credo.Check.Refactor.LongQuoteBlocks
     quote do
@@ -206,6 +206,19 @@ defmodule Explorer.Chain.OrderedCache do
 
       ### Updating function
 
+      def remove_deleted_from_index({:delete, _cache_pid, id}) do
+        # simply check with `ConCache.get` because it is faster
+        if Enum.member?(ids_list(), id) do
+          ConCache.update(cache_name(), ids_list_key(), fn ids ->
+            updated_list = List.delete(ids || [], id)
+            # ids_list is set to never expire
+            {:ok, %ConCache.Item{value: updated_list, ttl: :infinity}}
+          end)
+        end
+      end
+
+      def remove_deleted_from_index(_), do: nil
+
       @impl OrderedCache
       def update(elements) when is_nil(elements), do: :ok
 
@@ -217,7 +230,8 @@ defmodule Explorer.Chain.OrderedCache do
             |> Enum.sort(&prevails?(&1, &2))
             |> merge_and_update(ids || [], max_size())
 
-          {:ok, updated_list}
+          # ids_list is set to never expire
+          {:ok, %ConCache.Item{value: updated_list, ttl: :infinity}}
         end)
       end
 
@@ -308,7 +322,21 @@ defmodule Explorer.Chain.OrderedCache do
       provided to this function will override the ones set by using the macro
       """
       def child_spec(params) do
-        params = Keyword.merge(unquote(concache_params), params)
+        # params specified in `use`
+        merged_params =
+          unquote(concache_params)
+          # params specified in `child_spec`
+          |> Keyword.merge(params)
+          # `:ttl_check_interval` needs to be specified, defaults to `false`
+          |> Keyword.put_new(:ttl_check_interval, false)
+
+        # if `:ttl_check_interval` is not `false` the expired values need to be
+        # removed from the cache's index
+        params =
+          case merged_params[:ttl_check_interval] do
+            false -> merged_params
+            _ -> Keyword.put_new(merged_params, :callback, &remove_deleted_from_index/1)
+          end
 
         Supervisor.child_spec({ConCache, params}, id: child_id())
       end

apps/explorer/test/explorer/chain/cache/block_number_test.exs

@@ -11,7 +11,7 @@ defmodule Explorer.Chain.Cache.BlockNumberTest do
     end)
   end
 
-  describe "get_max/1" do
+  describe "get_max/0" do
     test "returns max number" do
       insert(:block, number: 5)
 
@@ -19,7 +19,7 @@ defmodule Explorer.Chain.Cache.BlockNumberTest do
     end
   end
 
-  describe "get_min/1" do
+  describe "get_min/0" do
     test "returns min number" do
       insert(:block, number: 2)
 
@@ -27,7 +27,7 @@ defmodule Explorer.Chain.Cache.BlockNumberTest do
     end
   end
 
-  describe "get_all/1" do
+  describe "get_all/0" do
     test "returns min and max number" do
       insert(:block, number: 6)