Merge pull request #5369 from blockscout/vb-manage-memory-limit

Manage indexer memory limit

.dialyzer-ignore

@@ -34,4 +34,4 @@ lib/block_scout_web/views/transaction_view.ex:152
 lib/block_scout_web/views/transaction_view.ex:197
 lib/indexer/buffered_task.ex:402
 lib/indexer/buffered_task.ex:451
-lib/indexer/memory/monitor.ex:161
+lib/indexer/memory/monitor.ex:160

CHANGELOG.md

@@ -24,6 +24,7 @@
 - [#5239](https://github.com/blockscout/blockscout/pull/5239) - Add accounting for block rewards in `getblockreward` api method
 
 ### Chore
+- [#5369](https://github.com/blockscout/blockscout/pull/5369) - Manage indexer memory limit
 - [#5368](https://github.com/blockscout/blockscout/pull/5368) - Refactoring from SourcifyFilePathBackfiller
 - [#5367](https://github.com/blockscout/blockscout/pull/5367) - Resolve Prototype Pollution in minimist dependency
 - [#5366](https://github.com/blockscout/blockscout/pull/5366) - Fix Vyper smart-contract verification form tooltips

apps/block_scout_web/config/config.exs

@@ -1,9 +1,9 @@
 # This file is responsible for configuring your application
-# and its dependencies with the aid of the Mix.Config module.
+# and its dependencies with the aid of the Config module.
 #
 # This configuration file is loaded before any dependency and
 # is restricted to this project.
-use Mix.Config
+import Config
 
 # General application configuration
 config :block_scout_web,
@@ -183,4 +183,4 @@ config :hammer,
 
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
-import_config "#{Mix.env()}.exs"
+import_config "#{config_env()}.exs"

apps/block_scout_web/config/dev.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 # For development, we disable any cache and enable
 # debugging and code reloading.

apps/block_scout_web/config/prod.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 # For production, we often load configuration from external
 # sources, such as your system environment. For this reason,

apps/block_scout_web/config/test.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 config :block_scout_web, :sql_sandbox, true
 

apps/ethereum_jsonrpc/config/config.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 config :ethereum_jsonrpc, EthereumJSONRPC.RequestCoordinator,
   rolling_window_opts: [
@@ -36,4 +36,4 @@ config :logger, :ethereum_jsonrpc,
 
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
-import_config "#{Mix.env()}.exs"
+import_config "#{config_env()}.exs"

apps/ethereum_jsonrpc/config/dev.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 config :ethereum_jsonrpc, EthereumJSONRPC.Tracer, env: "dev", disabled?: true
 

apps/ethereum_jsonrpc/config/prod.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 config :ethereum_jsonrpc, EthereumJSONRPC.Tracer, env: "production", disabled?: true
 

apps/ethereum_jsonrpc/config/test.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 config :ethereum_jsonrpc, EthereumJSONRPC.RequestCoordinator,
   rolling_window_opts: [

apps/explorer/config/config.exs

@@ -1,9 +1,9 @@
 # This file is responsible for configuring your application
-# and its dependencies with the aid of the Mix.Config module.
+# and its dependencies with the aid of the Config module.
 #
 # This configuration file is loaded before any dependency and
 # is restricted to this project.
-use Mix.Config
+import Config
 
 # General application configuration
 config :explorer,
@@ -256,4 +256,4 @@ config :explorer, Explorer.ThirdPartyIntegrations.Sourcify,
 
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
-import_config "#{Mix.env()}.exs"
+import_config "#{config_env()}.exs"

apps/explorer/config/dev.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 database = if System.get_env("DATABASE_URL"), do: nil, else: "explorer_dev"
 hostname = if System.get_env("DATABASE_URL"), do: nil, else: "localhost"

apps/explorer/config/dev/arbitrum.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 config :explorer,
   json_rpc_named_arguments: [

apps/explorer/config/dev/besu.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 config :explorer,
   json_rpc_named_arguments: [

apps/explorer/config/dev/ganache.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 config :explorer,
   json_rpc_named_arguments: [

apps/explorer/config/dev/geth.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 config :explorer,
   json_rpc_named_arguments: [

apps/explorer/config/dev/parity.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 config :explorer,
   json_rpc_named_arguments: [

apps/explorer/config/dev/rsk.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 config :explorer,
   json_rpc_named_arguments: [

apps/explorer/config/prod.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 pool_size =
   if System.get_env("DATABASE_READ_ONLY_API_URL"),

apps/explorer/config/prod/arbitrum.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 config :explorer,
   json_rpc_named_arguments: [

apps/explorer/config/prod/besu.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 config :explorer,
   json_rpc_named_arguments: [

apps/explorer/config/prod/ganache.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 config :explorer,
   json_rpc_named_arguments: [

apps/explorer/config/prod/geth.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 config :explorer,
   json_rpc_named_arguments: [

apps/explorer/config/prod/parity.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 config :explorer,
   json_rpc_named_arguments: [

apps/explorer/config/prod/rsk.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 config :explorer,
   json_rpc_named_arguments: [

apps/explorer/config/test.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 # Lower hashing rounds for faster tests
 config :bcrypt_elixir, log_rounds: 4

apps/explorer/config/test/arbitrum.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 config :explorer,
   json_rpc_named_arguments: [

apps/explorer/config/test/besu.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 config :explorer,
   transport: EthereumJSONRPC.HTTP,

apps/explorer/config/test/ganache.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 config :explorer,
   json_rpc_named_arguments: [

apps/explorer/config/test/geth.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 config :explorer,
   json_rpc_named_arguments: [

apps/explorer/config/test/parity.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 config :explorer,
   transport: EthereumJSONRPC.HTTP,

apps/explorer/config/test/rsk.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 config :explorer,
   transport: EthereumJSONRPC.HTTP,

apps/indexer/config/config.exs

@@ -1,8 +1,6 @@
 # This file is responsible for configuring your application
-# and its dependencies with the aid of the Mix.Config module.
+# and its dependencies with the aid of the Config module.
-use Mix.Config
+import Config
-
-import Bitwise
 
 block_transformers = %{
   "clique" => Indexer.Transform.Blocks.Clique,
@@ -33,8 +31,6 @@ config :indexer,
   ecto_repos: [Explorer.Repo],
   metadata_updater_seconds_interval:
     String.to_integer(System.get_env("TOKEN_METADATA_UPDATE_INTERVAL") || "#{2 * 24 * 60 * 60}"),
-  # bytes
-  memory_limit: 1 <<< 30,
   first_block: System.get_env("FIRST_BLOCK") || "",
   last_block: System.get_env("LAST_BLOCK") || "",
   trace_first_block: System.get_env("TRACE_FIRST_BLOCK") || "",
@@ -96,4 +92,4 @@ config :logger, :indexer,
 
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
-import_config "#{Mix.env()}.exs"
+import_config "#{config_env()}.exs"

apps/indexer/config/dev.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 config :indexer, Indexer.Tracer, env: "dev", disabled?: true
 

apps/indexer/config/dev/arbitrum.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 config :indexer,
   block_interval: :timer.seconds(5),

apps/indexer/config/dev/besu.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 config :indexer,
   block_interval: :timer.seconds(5),

apps/indexer/config/dev/ganache.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 config :indexer,
   block_interval: :timer.seconds(5),

apps/indexer/config/dev/geth.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 config :indexer,
   block_interval: :timer.seconds(5),

apps/indexer/config/dev/parity.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 config :indexer,
   block_interval: :timer.seconds(5),

apps/indexer/config/dev/rsk.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 config :indexer,
   block_interval: :timer.seconds(5),

apps/indexer/config/prod.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 config :indexer, Indexer.Tracer, env: "production", disabled?: true
 

apps/indexer/config/prod/arbitrum.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 config :indexer,
   block_interval: :timer.seconds(5),

apps/indexer/config/prod/besu.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 config :indexer,
   block_interval: :timer.seconds(5),

apps/indexer/config/prod/ganache.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 config :indexer,
   block_interval: :timer.seconds(5),

apps/indexer/config/prod/geth.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 config :indexer,
   block_interval: :timer.seconds(5),

apps/indexer/config/prod/parity.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 config :indexer,
   block_interval: :timer.seconds(5),

apps/indexer/config/prod/rsk.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 config :indexer,
   block_interval: :timer.seconds(5),

apps/indexer/config/test.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 config :indexer, Indexer.Tracer, disabled?: false
 

apps/indexer/config/test/arbitrum.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 config :indexer,
   json_rpc_named_arguments: [

apps/indexer/config/test/besu.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 config :indexer,
   json_rpc_named_arguments: [

apps/indexer/config/test/ganache.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 config :indexer,
   json_rpc_named_arguments: [

apps/indexer/config/test/geth.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 config :indexer,
   json_rpc_named_arguments: [

apps/indexer/config/test/parity.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 config :indexer,
   json_rpc_named_arguments: [

apps/indexer/config/test/rsk.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 config :indexer,
   json_rpc_named_arguments: [

apps/indexer/lib/indexer/empty_blocks_sanitizer.ex

@@ -16,7 +16,7 @@ defmodule Indexer.EmptyBlocksSanitizer do
   alias Explorer.Chain.{Block, Transaction}
   alias Explorer.Chain.Import.Runner.Blocks
 
-  # unprocessed emty blocks to fetch at once
+  # unprocessed empty blocks to fetch at once
   @limit 1000
 
   @interval :timer.minutes(1)

apps/indexer/lib/indexer/memory/monitor.ex

@@ -10,12 +10,11 @@ defmodule Indexer.Memory.Monitor do
   require Bitwise
   require Logger
 
-  import Bitwise
   import Indexer.Logger, only: [process: 1]
 
   alias Indexer.Memory.Shrinkable
 
-  defstruct limit: 1 <<< 30,
+  defstruct limit: Application.get_env(:indexer, :memory_limit),
             timer_interval: :timer.minutes(1),
             timer_reference: nil,
             shrinkable_set: MapSet.new()

config/config.exs

@@ -1,13 +1,15 @@
 # This file is responsible for configuring your application
-# and its dependencies with the aid of the Mix.Config module.
+# and its dependencies with the aid of the Config module.
-use Mix.Config
+import Config
 
 # By default, the umbrella project as well as each child
 # application will require this configuration file, ensuring
 # they all use the same configuration. While one could
 # configure all applications here, we prefer to delegate
 # back to each application for organization purposes.
-import_config "../apps/*/config/config.exs"
+for config <- "../apps/*/config/config.exs" |> Path.expand(__DIR__) |> Path.wildcard() do
+  import_config config
+end
 
 config :phoenix, :json_library, Jason
 
@@ -60,4 +62,4 @@ config :logger, :error,
 
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
-import_config "#{Mix.env()}.exs"
+import_config "#{config_env()}.exs"

config/dev.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 # DO NOT make it `:debug` or all Ecto logs will be shown for indexer
 config :logger, :console, level: :info

config/prod.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 # Do not print debug messages in production
 

config/runtime.exs

@@ -0,0 +1,15 @@
+import Config
+
+import Bitwise
+
+indexer_memory_limit =
+  "INDEXER_MEMORY_LIMIT"
+  |> System.get_env("1")
+  |> Integer.parse()
+  |> case do
+    {integer, ""} -> integer
+    _ -> 1
+  end
+
+config :indexer,
+  memory_limit: indexer_memory_limit <<< 30

config/test.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 # Print only warnings and errors during test
 

docker/Makefile

@@ -392,6 +392,9 @@ endif
 ifdef COIN_BALANCE_ON_DEMAND_FETCHER_THRESHOLD_MINUTES
 	BLOCKSCOUT_CONTAINER_PARAMS += -e 'COIN_BALANCE_ON_DEMAND_FETCHER_THRESHOLD_MINUTES=$(COIN_BALANCE_ON_DEMAND_FETCHER_THRESHOLD_MINUTES)'
 endif
+ifdef INDEXER_MEMORY_LIMIT
+	BLOCKSCOUT_CONTAINER_PARAMS += -e 'INDEXER_MEMORY_LIMIT=$(INDEXER_MEMORY_LIMIT)'
+endif
 
 HAS_BLOCKSCOUT_IMAGE := $(shell docker images | grep -sw "${BS_CONTAINER_IMAGE} ")
 build: 

mix.exs

@@ -6,7 +6,7 @@ defmodule BlockScout.Mixfile do
   def project do
     [
       app: :block_scout,
-      aliases: aliases(Mix.env()),
+      # aliases: aliases(config_env()),
       version: "4.1.2",
       apps_path: "apps",
       deps: deps(),
@@ -16,7 +16,7 @@ defmodule BlockScout.Mixfile do
         credo: :test,
         dialyzer: :test
       ],
-      start_permanent: Mix.env() == :prod,
+      # start_permanent: config_env() == :prod,
       releases: [
         blockscout: [
           applications: [
@@ -42,23 +42,23 @@ defmodule BlockScout.Mixfile do
     ]
   end
 
-  defp aliases(env) do
+  # defp aliases(env) do
-    [
+  #   [
-      # to match behavior of `mix test` in `apps/indexer`, which needs to not start applications for `indexer` to
+  #     # to match behavior of `mix test` in `apps/indexer`, which needs to not start applications for `indexer` to
-      # prevent its supervision tree from starting, which is undesirable in test
+  #     # prevent its supervision tree from starting, which is undesirable in test
-      test: "test --no-start"
+  #     test: "test --no-start"
-    ] ++ env_aliases(env)
+  #   ] ++ env_aliases(env)
-  end
+  # end
 
-  defp env_aliases(:dev) do
+  # defp env_aliases(:dev) do
-    []
+  #   []
-  end
+  # end
 
-  defp env_aliases(_env) do
+  # defp env_aliases(_env) do
-    [
+  #   [
-      compile: "compile --warnings-as-errors"
+  #     compile: "compile --warnings-as-errors"
-    ]
+  #   ]
-  end
+  # end
 
   # Dependencies can be Hex packages:
   #

rel/config.exs

@@ -1,3 +1,5 @@
+import Config
+
 # Import all plugins from `rel/plugins`
 # They can then be used by adding `plugin MyPlugin` to
 # either an environment, or release definition, where
@@ -33,7 +35,7 @@ use Mix.Releases.Config,
     # This sets the default release built by `mix release`
     default_release: :default,
     # This sets the default environment used by `mix release`
-    default_environment: Mix.env()
+    default_environment: config_env()
 
 # For a full list of config options for both releases
 # and environments, visit https://hexdocs.pm/distillery/config/distillery.html