From f3e279d8534d3debd462f62bca7831627ff205bc Mon Sep 17 00:00:00 2001 From: Qwerty5Uiop <105209995+Qwerty5Uiop@users.noreply.github.com> Date: Fri, 18 Oct 2024 17:10:44 +0400 Subject: [PATCH] chore: Set indexer memory limit based on system info as a fallback (#10697) --- apps/indexer/config/config.exs | 5 +++ apps/indexer/lib/indexer/application.ex | 8 +---- apps/indexer/lib/indexer/memory/monitor.ex | 40 ++++++++++++++++------ apps/indexer/mix.exs | 2 +- config/config_helper.exs | 6 ++-- config/runtime.exs | 1 + cspell.json | 2 ++ docker-compose/envs/common-blockscout.env | 1 + 8 files changed, 42 insertions(+), 23 deletions(-) diff --git a/apps/indexer/config/config.exs b/apps/indexer/config/config.exs index 5ce9ab28a2..808c3671e3 100644 --- a/apps/indexer/config/config.exs +++ b/apps/indexer/config/config.exs @@ -20,6 +20,11 @@ config :logger, :indexer, block_number step count error_count shrunk import_id transaction_id)a, metadata_filter: [application: :indexer] +config :os_mon, + start_cpu_sup: false, + start_disksup: false, + start_memsup: true + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{config_env()}.exs" diff --git a/apps/indexer/lib/indexer/application.ex b/apps/indexer/lib/indexer/application.ex index 14ee265fbe..5851e73adb 100644 --- a/apps/indexer/lib/indexer/application.ex +++ b/apps/indexer/lib/indexer/application.ex @@ -15,12 +15,6 @@ defmodule Indexer.Application do @impl Application def start(_type, _args) do - memory_monitor_options = - case Application.get_env(:indexer, :memory_limit) do - nil -> %{} - integer when is_integer(integer) -> %{limit: integer} - end - memory_monitor_name = Memory.Monitor json_rpc_named_arguments = Application.fetch_env!(:indexer, :json_rpc_named_arguments) @@ -46,7 +40,7 @@ defmodule Indexer.Application do base_children = [ :hackney_pool.child_spec(:token_instance_fetcher, max_connections: pool_size), - {Memory.Monitor, [memory_monitor_options, [name: memory_monitor_name]]}, + {Memory.Monitor, [%{}, [name: memory_monitor_name]]}, {CoinBalanceOnDemand.Supervisor, [json_rpc_named_arguments]}, {ContractCodeOnDemand.Supervisor, [json_rpc_named_arguments]}, {TokenInstanceMetadataRefetchOnDemand.Supervisor, [json_rpc_named_arguments]}, diff --git a/apps/indexer/lib/indexer/memory/monitor.ex b/apps/indexer/lib/indexer/memory/monitor.ex index c907cda6b5..c4032f7632 100644 --- a/apps/indexer/lib/indexer/memory/monitor.ex +++ b/apps/indexer/lib/indexer/memory/monitor.ex @@ -7,9 +7,9 @@ defmodule Indexer.Memory.Monitor do `c:Indexer.Memory.Shrinkable.shrink/0`. """ - require Bitwise require Logger + import Bitwise import Indexer.Logger, only: [process: 1] alias Indexer.Memory.Shrinkable @@ -47,7 +47,7 @@ defmodule Indexer.Memory.Monitor do @impl GenServer def init(options) when is_map(options) do - state = struct!(__MODULE__, options) + state = struct!(__MODULE__, Map.put_new(options, :limit, define_memory_limit())) {:ok, timer_reference} = :timer.send_interval(state.timer_interval, :check) {:ok, %__MODULE__{state | timer_reference: timer_reference}} @@ -66,14 +66,14 @@ defmodule Indexer.Memory.Monitor do end @impl GenServer - def handle_info(:check, state) do + def handle_info(:check, %{limit: limit} = state) do total = :erlang.memory(:total) set_metrics(state) shrunk_state = - if memory_limit() < total do - log_memory(%{limit: memory_limit(), total: total}) + if limit < total do + log_memory(%{limit: limit, total: total}) shrink_or_log(state) %{state | shrunk?: true} else @@ -81,8 +81,8 @@ defmodule Indexer.Memory.Monitor do end final_state = - if state.shrunk? and total <= memory_limit() * @expandable_memory_coefficient do - log_expandable_memory(%{limit: memory_limit(), total: total}) + if state.shrunk? and total <= limit * @expandable_memory_coefficient do + log_expandable_memory(%{limit: limit, total: total}) expand(state) %{state | shrunk?: false} else @@ -94,6 +94,28 @@ defmodule Indexer.Memory.Monitor do {:noreply, final_state} end + defp define_memory_limit do + case Application.get_env(:indexer, :memory_limit) do + integer when is_integer(integer) -> integer + _not_set -> memory_limit_from_system() + end + end + + defp memory_limit_from_system do + default_limit = 1 <<< 30 + + percentage = + case Application.get_env(:explorer, :mode) do + :indexer -> 100 + :all -> Application.get_env(:indexer, :system_memory_percentage) + end + + case :memsup.get_system_memory_data()[:total_memory] do + nil -> default_limit + total_memory -> floor(total_memory * percentage / 100) + end + end + defp flush(message) do receive do ^message -> flush(message) @@ -250,8 +272,4 @@ defmodule Indexer.Memory.Monitor do |> Enum.map(fn pid -> {pid, memory(pid)} end) |> Enum.sort_by(&elem(&1, 1), &>=/2) end - - defp memory_limit do - Application.get_env(:indexer, :memory_limit) - end end diff --git a/apps/indexer/mix.exs b/apps/indexer/mix.exs index acfebe6c6a..3e2456724d 100644 --- a/apps/indexer/mix.exs +++ b/apps/indexer/mix.exs @@ -31,7 +31,7 @@ defmodule Indexer.MixProject do # Run "mix help compile.app" to learn about applications. def application do [ - extra_applications: [:logger], + extra_applications: [:logger, :os_mon], mod: {Indexer.Application, []} ] end diff --git a/config/config_helper.exs b/config/config_helper.exs index 6def45a00a..882ef3bfc7 100644 --- a/config/config_helper.exs +++ b/config/config_helper.exs @@ -160,16 +160,14 @@ defmodule ConfigHelper do @spec indexer_memory_limit() :: integer() def indexer_memory_limit do - indexer_memory_limit_default = 1 - "INDEXER_MEMORY_LIMIT" - |> safe_get_env(to_string(indexer_memory_limit_default)) + |> safe_get_env(nil) |> String.downcase() |> Integer.parse() |> case do {integer, g} when g in ["g", "gb", ""] -> integer <<< 30 {integer, m} when m in ["m", "mb"] -> integer <<< 20 - _ -> indexer_memory_limit_default <<< 30 + _ -> nil end end diff --git a/config/runtime.exs b/config/runtime.exs index e0be451262..caa9b9438e 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -673,6 +673,7 @@ config :indexer, trace_last_block: trace_last_block, fetch_rewards_way: System.get_env("FETCH_REWARDS_WAY", "trace_block"), memory_limit: ConfigHelper.indexer_memory_limit(), + system_memory_percentage: ConfigHelper.parse_integer_env_var("INDEXER_SYSTEM_MEMORY_PERCENTAGE", 60), receipts_batch_size: ConfigHelper.parse_integer_env_var("INDEXER_RECEIPTS_BATCH_SIZE", 250), receipts_concurrency: ConfigHelper.parse_integer_env_var("INDEXER_RECEIPTS_CONCURRENCY", 10), hide_indexing_progress_alert: ConfigHelper.parse_bool_env_var("INDEXER_HIDE_INDEXING_PROGRESS_ALERT"), diff --git a/cspell.json b/cspell.json index 3e2c5a3b11..fe3444f1df 100644 --- a/cspell.json +++ b/cspell.json @@ -161,6 +161,7 @@ "describedby", "differenceby", "discordapp", + "disksup", "dropzone", "dxgd", "dyntsrohg", @@ -312,6 +313,7 @@ "mdef", "MDWW", "meer", + "memsup", "Mendonça", "Menlo", "mergeable", diff --git a/docker-compose/envs/common-blockscout.env b/docker-compose/envs/common-blockscout.env index dec8df0744..4c4bdafd85 100644 --- a/docker-compose/envs/common-blockscout.env +++ b/docker-compose/envs/common-blockscout.env @@ -280,6 +280,7 @@ INDEXER_DISABLE_INTERNAL_TRANSACTIONS_FETCHER=false # INDEXER_COIN_BALANCES_FETCHER_INIT_QUERY_LIMIT= # INDEXER_GRACEFUL_SHUTDOWN_PERIOD= # INDEXER_INTERNAL_TRANSACTIONS_FETCH_ORDER= +# INDEXER_SYSTEM_MEMORY_PERCENTAGE= # WITHDRAWALS_FIRST_BLOCK= # INDEXER_OPTIMISM_L1_RPC= # INDEXER_OPTIMISM_L1_SYSTEM_CONFIG_CONTRACT=