chore: Set indexer memory limit based on system info as a fallback (#10697)

apps/indexer/config/config.exs

@@ -20,6 +20,11 @@ config :logger, :indexer,
        block_number step count error_count shrunk import_id transaction_id)a,
   metadata_filter: [application: :indexer]
 
+config :os_mon,
+  start_cpu_sup: false,
+  start_disksup: false,
+  start_memsup: true
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{config_env()}.exs"

apps/indexer/lib/indexer/application.ex

@@ -15,12 +15,6 @@ defmodule Indexer.Application do
 
   @impl Application
   def start(_type, _args) do
-    memory_monitor_options =
-      case Application.get_env(:indexer, :memory_limit) do
-        nil -> %{}
-        integer when is_integer(integer) -> %{limit: integer}
-      end
-
     memory_monitor_name = Memory.Monitor
 
     json_rpc_named_arguments = Application.fetch_env!(:indexer, :json_rpc_named_arguments)
@@ -46,7 +40,7 @@ defmodule Indexer.Application do
 
     base_children = [
       :hackney_pool.child_spec(:token_instance_fetcher, max_connections: pool_size),
-      {Memory.Monitor, [memory_monitor_options, [name: memory_monitor_name]]},
+      {Memory.Monitor, [%{}, [name: memory_monitor_name]]},
       {CoinBalanceOnDemand.Supervisor, [json_rpc_named_arguments]},
       {ContractCodeOnDemand.Supervisor, [json_rpc_named_arguments]},
       {TokenInstanceMetadataRefetchOnDemand.Supervisor, [json_rpc_named_arguments]},

apps/indexer/lib/indexer/memory/monitor.ex

@@ -7,9 +7,9 @@ defmodule Indexer.Memory.Monitor do
   `c:Indexer.Memory.Shrinkable.shrink/0`.
   """
 
-  require Bitwise
   require Logger
 
+  import Bitwise
   import Indexer.Logger, only: [process: 1]
 
   alias Indexer.Memory.Shrinkable
@@ -47,7 +47,7 @@ defmodule Indexer.Memory.Monitor do
 
   @impl GenServer
   def init(options) when is_map(options) do
-    state = struct!(__MODULE__, options)
+    state = struct!(__MODULE__, Map.put_new(options, :limit, define_memory_limit()))
     {:ok, timer_reference} = :timer.send_interval(state.timer_interval, :check)
 
     {:ok, %__MODULE__{state | timer_reference: timer_reference}}
@@ -66,14 +66,14 @@ defmodule Indexer.Memory.Monitor do
   end
 
   @impl GenServer
-  def handle_info(:check, state) do
+  def handle_info(:check, %{limit: limit} = state) do
     total = :erlang.memory(:total)
 
     set_metrics(state)
 
     shrunk_state =
-      if memory_limit() < total do
+      if limit < total do
-        log_memory(%{limit: memory_limit(), total: total})
+        log_memory(%{limit: limit, total: total})
         shrink_or_log(state)
         %{state | shrunk?: true}
       else
@@ -81,8 +81,8 @@ defmodule Indexer.Memory.Monitor do
       end
 
     final_state =
-      if state.shrunk? and total <= memory_limit() * @expandable_memory_coefficient do
+      if state.shrunk? and total <= limit * @expandable_memory_coefficient do
-        log_expandable_memory(%{limit: memory_limit(), total: total})
+        log_expandable_memory(%{limit: limit, total: total})
         expand(state)
         %{state | shrunk?: false}
       else
@@ -94,6 +94,28 @@ defmodule Indexer.Memory.Monitor do
     {:noreply, final_state}
   end
 
+  defp define_memory_limit do
+    case Application.get_env(:indexer, :memory_limit) do
+      integer when is_integer(integer) -> integer
+      _not_set -> memory_limit_from_system()
+    end
+  end
+
+  defp memory_limit_from_system do
+    default_limit = 1 <<< 30
+
+    percentage =
+      case Application.get_env(:explorer, :mode) do
+        :indexer -> 100
+        :all -> Application.get_env(:indexer, :system_memory_percentage)
+      end
+
+    case :memsup.get_system_memory_data()[:total_memory] do
+      nil -> default_limit
+      total_memory -> floor(total_memory * percentage / 100)
+    end
+  end
+
   defp flush(message) do
     receive do
       ^message -> flush(message)
@@ -250,8 +272,4 @@ defmodule Indexer.Memory.Monitor do
     |> Enum.map(fn pid -> {pid, memory(pid)} end)
     |> Enum.sort_by(&elem(&1, 1), &>=/2)
   end
-
-  defp memory_limit do
-    Application.get_env(:indexer, :memory_limit)
-  end
 end

apps/indexer/mix.exs

@@ -31,7 +31,7 @@ defmodule Indexer.MixProject do
   # Run "mix help compile.app" to learn about applications.
   def application do
     [
-      extra_applications: [:logger],
+      extra_applications: [:logger, :os_mon],
       mod: {Indexer.Application, []}
     ]
   end

config/config_helper.exs

@@ -160,16 +160,14 @@ defmodule ConfigHelper do
 
   @spec indexer_memory_limit() :: integer()
   def indexer_memory_limit do
-    indexer_memory_limit_default = 1
-
     "INDEXER_MEMORY_LIMIT"
-    |> safe_get_env(to_string(indexer_memory_limit_default))
+    |> safe_get_env(nil)
     |> String.downcase()
     |> Integer.parse()
     |> case do
       {integer, g} when g in ["g", "gb", ""] -> integer <<< 30
       {integer, m} when m in ["m", "mb"] -> integer <<< 20
-      _ -> indexer_memory_limit_default <<< 30
+      _ -> nil
     end
   end
 

config/runtime.exs

@@ -673,6 +673,7 @@ config :indexer,
   trace_last_block: trace_last_block,
   fetch_rewards_way: System.get_env("FETCH_REWARDS_WAY", "trace_block"),
   memory_limit: ConfigHelper.indexer_memory_limit(),
+  system_memory_percentage: ConfigHelper.parse_integer_env_var("INDEXER_SYSTEM_MEMORY_PERCENTAGE", 60),
   receipts_batch_size: ConfigHelper.parse_integer_env_var("INDEXER_RECEIPTS_BATCH_SIZE", 250),
   receipts_concurrency: ConfigHelper.parse_integer_env_var("INDEXER_RECEIPTS_CONCURRENCY", 10),
   hide_indexing_progress_alert: ConfigHelper.parse_bool_env_var("INDEXER_HIDE_INDEXING_PROGRESS_ALERT"),

cspell.json

@@ -161,6 +161,7 @@
         "describedby",
         "differenceby",
         "discordapp",
+        "disksup",
         "dropzone",
         "dxgd",
         "dyntsrohg",
@@ -312,6 +313,7 @@
         "mdef",
         "MDWW",
         "meer",
+        "memsup",
         "Mendonça",
         "Menlo",
         "mergeable",

docker-compose/envs/common-blockscout.env

@@ -280,6 +280,7 @@ INDEXER_DISABLE_INTERNAL_TRANSACTIONS_FETCHER=false
 # INDEXER_COIN_BALANCES_FETCHER_INIT_QUERY_LIMIT=
 # INDEXER_GRACEFUL_SHUTDOWN_PERIOD=
 # INDEXER_INTERNAL_TRANSACTIONS_FETCH_ORDER=
+# INDEXER_SYSTEM_MEMORY_PERCENTAGE=
 # WITHDRAWALS_FIRST_BLOCK=
 # INDEXER_OPTIMISM_L1_RPC=
 # INDEXER_OPTIMISM_L1_SYSTEM_CONFIG_CONTRACT=