From bf3a7869e4c112841a2b4524352d7efcaecd195e Mon Sep 17 00:00:00 2001 From: Viktor Baranov Date: Fri, 25 Mar 2022 11:34:03 +0300 Subject: [PATCH] Manage indexer memory limit --- .dialyzer-ignore | 2 +- CHANGELOG.md | 1 + apps/block_scout_web/config/config.exs | 6 ++-- apps/block_scout_web/config/dev.exs | 2 +- apps/block_scout_web/config/prod.exs | 2 +- apps/block_scout_web/config/test.exs | 2 +- apps/ethereum_jsonrpc/config/config.exs | 4 +-- apps/ethereum_jsonrpc/config/dev.exs | 2 +- apps/ethereum_jsonrpc/config/prod.exs | 2 +- apps/ethereum_jsonrpc/config/test.exs | 2 +- apps/explorer/config/config.exs | 6 ++-- apps/explorer/config/dev.exs | 2 +- apps/explorer/config/dev/arbitrum.exs | 2 +- apps/explorer/config/dev/besu.exs | 2 +- apps/explorer/config/dev/ganache.exs | 2 +- apps/explorer/config/dev/geth.exs | 2 +- apps/explorer/config/dev/parity.exs | 2 +- apps/explorer/config/dev/rsk.exs | 2 +- apps/explorer/config/prod.exs | 2 +- apps/explorer/config/prod/arbitrum.exs | 2 +- apps/explorer/config/prod/besu.exs | 2 +- apps/explorer/config/prod/ganache.exs | 2 +- apps/explorer/config/prod/geth.exs | 2 +- apps/explorer/config/prod/parity.exs | 2 +- apps/explorer/config/prod/rsk.exs | 2 +- apps/explorer/config/test.exs | 2 +- apps/explorer/config/test/arbitrum.exs | 2 +- apps/explorer/config/test/besu.exs | 2 +- apps/explorer/config/test/ganache.exs | 2 +- apps/explorer/config/test/geth.exs | 2 +- apps/explorer/config/test/parity.exs | 2 +- apps/explorer/config/test/rsk.exs | 2 +- apps/indexer/config/config.exs | 10 ++---- apps/indexer/config/dev.exs | 2 +- apps/indexer/config/dev/arbitrum.exs | 2 +- apps/indexer/config/dev/besu.exs | 2 +- apps/indexer/config/dev/ganache.exs | 2 +- apps/indexer/config/dev/geth.exs | 2 +- apps/indexer/config/dev/parity.exs | 2 +- apps/indexer/config/dev/rsk.exs | 2 +- apps/indexer/config/prod.exs | 2 +- apps/indexer/config/prod/arbitrum.exs | 2 +- apps/indexer/config/prod/besu.exs | 2 +- apps/indexer/config/prod/ganache.exs | 2 +- apps/indexer/config/prod/geth.exs | 2 +- apps/indexer/config/prod/parity.exs | 2 +- apps/indexer/config/prod/rsk.exs | 2 +- apps/indexer/config/test.exs | 2 +- apps/indexer/config/test/arbitrum.exs | 2 +- apps/indexer/config/test/besu.exs | 2 +- apps/indexer/config/test/ganache.exs | 2 +- apps/indexer/config/test/geth.exs | 2 +- apps/indexer/config/test/parity.exs | 2 +- apps/indexer/config/test/rsk.exs | 2 +- .../lib/indexer/empty_blocks_sanitizer.ex | 2 +- apps/indexer/lib/indexer/memory/monitor.ex | 3 +- config/config.exs | 10 +++--- config/dev.exs | 2 +- config/prod.exs | 2 +- config/runtime.exs | 15 ++++++++ config/test.exs | 2 +- docker/Makefile | 3 ++ mix.exs | 34 +++++++++---------- rel/config.exs | 4 ++- 64 files changed, 110 insertions(+), 92 deletions(-) create mode 100644 config/runtime.exs diff --git a/.dialyzer-ignore b/.dialyzer-ignore index d45a57f55c..dcd9c797dc 100644 --- a/.dialyzer-ignore +++ b/.dialyzer-ignore @@ -34,4 +34,4 @@ lib/block_scout_web/views/transaction_view.ex:152 lib/block_scout_web/views/transaction_view.ex:197 lib/indexer/buffered_task.ex:402 lib/indexer/buffered_task.ex:451 -lib/indexer/memory/monitor.ex:161 +lib/indexer/memory/monitor.ex:160 diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ab3642624..002afa765c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ - [#5239](https://github.com/blockscout/blockscout/pull/5239) - Add accounting for block rewards in `getblockreward` api method ### Chore +- [#5369](https://github.com/blockscout/blockscout/pull/5369) - Manage indexer memory limit - [#5368](https://github.com/blockscout/blockscout/pull/5368) - Refactoring from SourcifyFilePathBackfiller - [#5367](https://github.com/blockscout/blockscout/pull/5367) - Resolve Prototype Pollution in minimist dependency - [#5366](https://github.com/blockscout/blockscout/pull/5366) - Fix Vyper smart-contract verification form tooltips diff --git a/apps/block_scout_web/config/config.exs b/apps/block_scout_web/config/config.exs index 51d2db9ba8..8392cfdda8 100644 --- a/apps/block_scout_web/config/config.exs +++ b/apps/block_scout_web/config/config.exs @@ -1,9 +1,9 @@ # This file is responsible for configuring your application -# and its dependencies with the aid of the Mix.Config module. +# and its dependencies with the aid of the Config module. # # This configuration file is loaded before any dependency and # is restricted to this project. -use Mix.Config +import Config # General application configuration config :block_scout_web, @@ -183,4 +183,4 @@ config :hammer, # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. -import_config "#{Mix.env()}.exs" +import_config "#{config_env()}.exs" diff --git a/apps/block_scout_web/config/dev.exs b/apps/block_scout_web/config/dev.exs index 82f9fc6629..02b038790c 100644 --- a/apps/block_scout_web/config/dev.exs +++ b/apps/block_scout_web/config/dev.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config # For development, we disable any cache and enable # debugging and code reloading. diff --git a/apps/block_scout_web/config/prod.exs b/apps/block_scout_web/config/prod.exs index 2277d38ef0..fd433c47f1 100644 --- a/apps/block_scout_web/config/prod.exs +++ b/apps/block_scout_web/config/prod.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config # For production, we often load configuration from external # sources, such as your system environment. For this reason, diff --git a/apps/block_scout_web/config/test.exs b/apps/block_scout_web/config/test.exs index 967ee24c70..a2188c180c 100644 --- a/apps/block_scout_web/config/test.exs +++ b/apps/block_scout_web/config/test.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config config :block_scout_web, :sql_sandbox, true diff --git a/apps/ethereum_jsonrpc/config/config.exs b/apps/ethereum_jsonrpc/config/config.exs index 9a564a1e80..251b7c7e3a 100644 --- a/apps/ethereum_jsonrpc/config/config.exs +++ b/apps/ethereum_jsonrpc/config/config.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config config :ethereum_jsonrpc, EthereumJSONRPC.RequestCoordinator, rolling_window_opts: [ @@ -36,4 +36,4 @@ config :logger, :ethereum_jsonrpc, # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. -import_config "#{Mix.env()}.exs" +import_config "#{config_env()}.exs" diff --git a/apps/ethereum_jsonrpc/config/dev.exs b/apps/ethereum_jsonrpc/config/dev.exs index 1f81a56cb5..da39fcf080 100644 --- a/apps/ethereum_jsonrpc/config/dev.exs +++ b/apps/ethereum_jsonrpc/config/dev.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config config :ethereum_jsonrpc, EthereumJSONRPC.Tracer, env: "dev", disabled?: true diff --git a/apps/ethereum_jsonrpc/config/prod.exs b/apps/ethereum_jsonrpc/config/prod.exs index 4b4d80babb..f11e4e7a72 100644 --- a/apps/ethereum_jsonrpc/config/prod.exs +++ b/apps/ethereum_jsonrpc/config/prod.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config config :ethereum_jsonrpc, EthereumJSONRPC.Tracer, env: "production", disabled?: true diff --git a/apps/ethereum_jsonrpc/config/test.exs b/apps/ethereum_jsonrpc/config/test.exs index 88e4eba302..0ed3de28b2 100644 --- a/apps/ethereum_jsonrpc/config/test.exs +++ b/apps/ethereum_jsonrpc/config/test.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config config :ethereum_jsonrpc, EthereumJSONRPC.RequestCoordinator, rolling_window_opts: [ diff --git a/apps/explorer/config/config.exs b/apps/explorer/config/config.exs index f83a14c5e7..6fe9b7816e 100644 --- a/apps/explorer/config/config.exs +++ b/apps/explorer/config/config.exs @@ -1,9 +1,9 @@ # This file is responsible for configuring your application -# and its dependencies with the aid of the Mix.Config module. +# and its dependencies with the aid of the Config module. # # This configuration file is loaded before any dependency and # is restricted to this project. -use Mix.Config +import Config # General application configuration config :explorer, @@ -256,4 +256,4 @@ config :explorer, Explorer.ThirdPartyIntegrations.Sourcify, # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. -import_config "#{Mix.env()}.exs" +import_config "#{config_env()}.exs" diff --git a/apps/explorer/config/dev.exs b/apps/explorer/config/dev.exs index c37940c717..552e5d29a1 100644 --- a/apps/explorer/config/dev.exs +++ b/apps/explorer/config/dev.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config database = if System.get_env("DATABASE_URL"), do: nil, else: "explorer_dev" hostname = if System.get_env("DATABASE_URL"), do: nil, else: "localhost" diff --git a/apps/explorer/config/dev/arbitrum.exs b/apps/explorer/config/dev/arbitrum.exs index 1c80b41765..e3996f056d 100644 --- a/apps/explorer/config/dev/arbitrum.exs +++ b/apps/explorer/config/dev/arbitrum.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config config :explorer, json_rpc_named_arguments: [ diff --git a/apps/explorer/config/dev/besu.exs b/apps/explorer/config/dev/besu.exs index e014ac960c..a277e2c68e 100644 --- a/apps/explorer/config/dev/besu.exs +++ b/apps/explorer/config/dev/besu.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config config :explorer, json_rpc_named_arguments: [ diff --git a/apps/explorer/config/dev/ganache.exs b/apps/explorer/config/dev/ganache.exs index e6c2214601..dcea3bb049 100644 --- a/apps/explorer/config/dev/ganache.exs +++ b/apps/explorer/config/dev/ganache.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config config :explorer, json_rpc_named_arguments: [ diff --git a/apps/explorer/config/dev/geth.exs b/apps/explorer/config/dev/geth.exs index f6a51993c2..f5ba58f1e4 100644 --- a/apps/explorer/config/dev/geth.exs +++ b/apps/explorer/config/dev/geth.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config config :explorer, json_rpc_named_arguments: [ diff --git a/apps/explorer/config/dev/parity.exs b/apps/explorer/config/dev/parity.exs index aebe0dfac1..4008d5eb6d 100644 --- a/apps/explorer/config/dev/parity.exs +++ b/apps/explorer/config/dev/parity.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config config :explorer, json_rpc_named_arguments: [ diff --git a/apps/explorer/config/dev/rsk.exs b/apps/explorer/config/dev/rsk.exs index 58af2ee8c0..2e4a9da460 100644 --- a/apps/explorer/config/dev/rsk.exs +++ b/apps/explorer/config/dev/rsk.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config config :explorer, json_rpc_named_arguments: [ diff --git a/apps/explorer/config/prod.exs b/apps/explorer/config/prod.exs index c12dce364b..cf1a2d010a 100644 --- a/apps/explorer/config/prod.exs +++ b/apps/explorer/config/prod.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config pool_size = if System.get_env("DATABASE_READ_ONLY_API_URL"), diff --git a/apps/explorer/config/prod/arbitrum.exs b/apps/explorer/config/prod/arbitrum.exs index 8ef7b06dbb..3663189d3d 100644 --- a/apps/explorer/config/prod/arbitrum.exs +++ b/apps/explorer/config/prod/arbitrum.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config config :explorer, json_rpc_named_arguments: [ diff --git a/apps/explorer/config/prod/besu.exs b/apps/explorer/config/prod/besu.exs index 4b69e1363e..8eaada3971 100644 --- a/apps/explorer/config/prod/besu.exs +++ b/apps/explorer/config/prod/besu.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config config :explorer, json_rpc_named_arguments: [ diff --git a/apps/explorer/config/prod/ganache.exs b/apps/explorer/config/prod/ganache.exs index c0a4b98a02..694d1ea4d9 100644 --- a/apps/explorer/config/prod/ganache.exs +++ b/apps/explorer/config/prod/ganache.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config config :explorer, json_rpc_named_arguments: [ diff --git a/apps/explorer/config/prod/geth.exs b/apps/explorer/config/prod/geth.exs index 026d745e9b..026c062047 100644 --- a/apps/explorer/config/prod/geth.exs +++ b/apps/explorer/config/prod/geth.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config config :explorer, json_rpc_named_arguments: [ diff --git a/apps/explorer/config/prod/parity.exs b/apps/explorer/config/prod/parity.exs index 68c1d14fdf..796a066433 100644 --- a/apps/explorer/config/prod/parity.exs +++ b/apps/explorer/config/prod/parity.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config config :explorer, json_rpc_named_arguments: [ diff --git a/apps/explorer/config/prod/rsk.exs b/apps/explorer/config/prod/rsk.exs index fc62a1c431..316be82dbf 100644 --- a/apps/explorer/config/prod/rsk.exs +++ b/apps/explorer/config/prod/rsk.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config config :explorer, json_rpc_named_arguments: [ diff --git a/apps/explorer/config/test.exs b/apps/explorer/config/test.exs index d499642ceb..becfc8e490 100644 --- a/apps/explorer/config/test.exs +++ b/apps/explorer/config/test.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config # Lower hashing rounds for faster tests config :bcrypt_elixir, log_rounds: 4 diff --git a/apps/explorer/config/test/arbitrum.exs b/apps/explorer/config/test/arbitrum.exs index 5a910d5417..2bb92bef12 100644 --- a/apps/explorer/config/test/arbitrum.exs +++ b/apps/explorer/config/test/arbitrum.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config config :explorer, json_rpc_named_arguments: [ diff --git a/apps/explorer/config/test/besu.exs b/apps/explorer/config/test/besu.exs index f5c234c7e1..25b35515b4 100644 --- a/apps/explorer/config/test/besu.exs +++ b/apps/explorer/config/test/besu.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config config :explorer, transport: EthereumJSONRPC.HTTP, diff --git a/apps/explorer/config/test/ganache.exs b/apps/explorer/config/test/ganache.exs index abb61a0632..a73d451d40 100644 --- a/apps/explorer/config/test/ganache.exs +++ b/apps/explorer/config/test/ganache.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config config :explorer, json_rpc_named_arguments: [ diff --git a/apps/explorer/config/test/geth.exs b/apps/explorer/config/test/geth.exs index 73869e31a3..ba67923248 100644 --- a/apps/explorer/config/test/geth.exs +++ b/apps/explorer/config/test/geth.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config config :explorer, json_rpc_named_arguments: [ diff --git a/apps/explorer/config/test/parity.exs b/apps/explorer/config/test/parity.exs index 0859de439a..c0e9464f7e 100644 --- a/apps/explorer/config/test/parity.exs +++ b/apps/explorer/config/test/parity.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config config :explorer, transport: EthereumJSONRPC.HTTP, diff --git a/apps/explorer/config/test/rsk.exs b/apps/explorer/config/test/rsk.exs index 861525bb0d..1799a52dc4 100644 --- a/apps/explorer/config/test/rsk.exs +++ b/apps/explorer/config/test/rsk.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config config :explorer, transport: EthereumJSONRPC.HTTP, diff --git a/apps/indexer/config/config.exs b/apps/indexer/config/config.exs index 9b39dd5cc8..225fe6978a 100644 --- a/apps/indexer/config/config.exs +++ b/apps/indexer/config/config.exs @@ -1,8 +1,6 @@ # This file is responsible for configuring your application -# and its dependencies with the aid of the Mix.Config module. -use Mix.Config - -import Bitwise +# and its dependencies with the aid of the Config module. +import Config block_transformers = %{ "clique" => Indexer.Transform.Blocks.Clique, @@ -33,8 +31,6 @@ config :indexer, ecto_repos: [Explorer.Repo], metadata_updater_seconds_interval: String.to_integer(System.get_env("TOKEN_METADATA_UPDATE_INTERVAL") || "#{2 * 24 * 60 * 60}"), - # bytes - memory_limit: 1 <<< 30, first_block: System.get_env("FIRST_BLOCK") || "", last_block: System.get_env("LAST_BLOCK") || "", trace_first_block: System.get_env("TRACE_FIRST_BLOCK") || "", @@ -96,4 +92,4 @@ config :logger, :indexer, # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. -import_config "#{Mix.env()}.exs" +import_config "#{config_env()}.exs" diff --git a/apps/indexer/config/dev.exs b/apps/indexer/config/dev.exs index 02061549a8..7124cbf521 100644 --- a/apps/indexer/config/dev.exs +++ b/apps/indexer/config/dev.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config config :indexer, Indexer.Tracer, env: "dev", disabled?: true diff --git a/apps/indexer/config/dev/arbitrum.exs b/apps/indexer/config/dev/arbitrum.exs index f56213ae01..e0f8de63e2 100644 --- a/apps/indexer/config/dev/arbitrum.exs +++ b/apps/indexer/config/dev/arbitrum.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config config :indexer, block_interval: :timer.seconds(5), diff --git a/apps/indexer/config/dev/besu.exs b/apps/indexer/config/dev/besu.exs index 995adbfb71..af23f64718 100644 --- a/apps/indexer/config/dev/besu.exs +++ b/apps/indexer/config/dev/besu.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config config :indexer, block_interval: :timer.seconds(5), diff --git a/apps/indexer/config/dev/ganache.exs b/apps/indexer/config/dev/ganache.exs index 229d9d36d9..ee377b5022 100644 --- a/apps/indexer/config/dev/ganache.exs +++ b/apps/indexer/config/dev/ganache.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config config :indexer, block_interval: :timer.seconds(5), diff --git a/apps/indexer/config/dev/geth.exs b/apps/indexer/config/dev/geth.exs index 3082fbebf7..e2f26eca47 100644 --- a/apps/indexer/config/dev/geth.exs +++ b/apps/indexer/config/dev/geth.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config config :indexer, block_interval: :timer.seconds(5), diff --git a/apps/indexer/config/dev/parity.exs b/apps/indexer/config/dev/parity.exs index bff27731f0..b3e14583d7 100644 --- a/apps/indexer/config/dev/parity.exs +++ b/apps/indexer/config/dev/parity.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config config :indexer, block_interval: :timer.seconds(5), diff --git a/apps/indexer/config/dev/rsk.exs b/apps/indexer/config/dev/rsk.exs index 28c19e6c7f..8132cb7cd6 100644 --- a/apps/indexer/config/dev/rsk.exs +++ b/apps/indexer/config/dev/rsk.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config config :indexer, block_interval: :timer.seconds(5), diff --git a/apps/indexer/config/prod.exs b/apps/indexer/config/prod.exs index 76f4bcac52..0fa704c5a6 100644 --- a/apps/indexer/config/prod.exs +++ b/apps/indexer/config/prod.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config config :indexer, Indexer.Tracer, env: "production", disabled?: true diff --git a/apps/indexer/config/prod/arbitrum.exs b/apps/indexer/config/prod/arbitrum.exs index dffc501d35..b2780524c3 100644 --- a/apps/indexer/config/prod/arbitrum.exs +++ b/apps/indexer/config/prod/arbitrum.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config config :indexer, block_interval: :timer.seconds(5), diff --git a/apps/indexer/config/prod/besu.exs b/apps/indexer/config/prod/besu.exs index c6393935fe..70360adc1a 100644 --- a/apps/indexer/config/prod/besu.exs +++ b/apps/indexer/config/prod/besu.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config config :indexer, block_interval: :timer.seconds(5), diff --git a/apps/indexer/config/prod/ganache.exs b/apps/indexer/config/prod/ganache.exs index 229d9d36d9..ee377b5022 100644 --- a/apps/indexer/config/prod/ganache.exs +++ b/apps/indexer/config/prod/ganache.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config config :indexer, block_interval: :timer.seconds(5), diff --git a/apps/indexer/config/prod/geth.exs b/apps/indexer/config/prod/geth.exs index 85d8563ce9..4de0919c2b 100644 --- a/apps/indexer/config/prod/geth.exs +++ b/apps/indexer/config/prod/geth.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config config :indexer, block_interval: :timer.seconds(5), diff --git a/apps/indexer/config/prod/parity.exs b/apps/indexer/config/prod/parity.exs index 8dec293444..e9dde34814 100644 --- a/apps/indexer/config/prod/parity.exs +++ b/apps/indexer/config/prod/parity.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config config :indexer, block_interval: :timer.seconds(5), diff --git a/apps/indexer/config/prod/rsk.exs b/apps/indexer/config/prod/rsk.exs index 5d52ad470d..daa5f22454 100644 --- a/apps/indexer/config/prod/rsk.exs +++ b/apps/indexer/config/prod/rsk.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config config :indexer, block_interval: :timer.seconds(5), diff --git a/apps/indexer/config/test.exs b/apps/indexer/config/test.exs index e74b5e1abf..c1fb6c0c55 100644 --- a/apps/indexer/config/test.exs +++ b/apps/indexer/config/test.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config config :indexer, Indexer.Tracer, disabled?: false diff --git a/apps/indexer/config/test/arbitrum.exs b/apps/indexer/config/test/arbitrum.exs index e2e51342d1..717b3f730b 100644 --- a/apps/indexer/config/test/arbitrum.exs +++ b/apps/indexer/config/test/arbitrum.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config config :indexer, json_rpc_named_arguments: [ diff --git a/apps/indexer/config/test/besu.exs b/apps/indexer/config/test/besu.exs index ceaac3a9b4..2d388f1702 100644 --- a/apps/indexer/config/test/besu.exs +++ b/apps/indexer/config/test/besu.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config config :indexer, json_rpc_named_arguments: [ diff --git a/apps/indexer/config/test/ganache.exs b/apps/indexer/config/test/ganache.exs index b421779152..9fa1d7e199 100644 --- a/apps/indexer/config/test/ganache.exs +++ b/apps/indexer/config/test/ganache.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config config :indexer, json_rpc_named_arguments: [ diff --git a/apps/indexer/config/test/geth.exs b/apps/indexer/config/test/geth.exs index 7c4b58b31a..c70674d063 100644 --- a/apps/indexer/config/test/geth.exs +++ b/apps/indexer/config/test/geth.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config config :indexer, json_rpc_named_arguments: [ diff --git a/apps/indexer/config/test/parity.exs b/apps/indexer/config/test/parity.exs index af3c396e9a..217587522a 100644 --- a/apps/indexer/config/test/parity.exs +++ b/apps/indexer/config/test/parity.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config config :indexer, json_rpc_named_arguments: [ diff --git a/apps/indexer/config/test/rsk.exs b/apps/indexer/config/test/rsk.exs index 15e2768fc7..998275b2df 100644 --- a/apps/indexer/config/test/rsk.exs +++ b/apps/indexer/config/test/rsk.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config config :indexer, json_rpc_named_arguments: [ diff --git a/apps/indexer/lib/indexer/empty_blocks_sanitizer.ex b/apps/indexer/lib/indexer/empty_blocks_sanitizer.ex index 2e7ed7ee1f..8551045e93 100644 --- a/apps/indexer/lib/indexer/empty_blocks_sanitizer.ex +++ b/apps/indexer/lib/indexer/empty_blocks_sanitizer.ex @@ -16,7 +16,7 @@ defmodule Indexer.EmptyBlocksSanitizer do alias Explorer.Chain.{Block, Transaction} alias Explorer.Chain.Import.Runner.Blocks - # unprocessed emty blocks to fetch at once + # unprocessed empty blocks to fetch at once @limit 1000 @interval :timer.minutes(1) diff --git a/apps/indexer/lib/indexer/memory/monitor.ex b/apps/indexer/lib/indexer/memory/monitor.ex index 3ee826439b..a63d039be2 100644 --- a/apps/indexer/lib/indexer/memory/monitor.ex +++ b/apps/indexer/lib/indexer/memory/monitor.ex @@ -10,12 +10,11 @@ defmodule Indexer.Memory.Monitor do require Bitwise require Logger - import Bitwise import Indexer.Logger, only: [process: 1] alias Indexer.Memory.Shrinkable - defstruct limit: 1 <<< 30, + defstruct limit: Application.get_env(:indexer, :memory_limit), timer_interval: :timer.minutes(1), timer_reference: nil, shrinkable_set: MapSet.new() diff --git a/config/config.exs b/config/config.exs index 939702dade..edd96d5c06 100644 --- a/config/config.exs +++ b/config/config.exs @@ -1,13 +1,15 @@ # This file is responsible for configuring your application -# and its dependencies with the aid of the Mix.Config module. -use Mix.Config +# and its dependencies with the aid of the Config module. +import Config # By default, the umbrella project as well as each child # application will require this configuration file, ensuring # they all use the same configuration. While one could # configure all applications here, we prefer to delegate # back to each application for organization purposes. -import_config "../apps/*/config/config.exs" +for config <- "../apps/*/config/config.exs" |> Path.expand(__DIR__) |> Path.wildcard() do + import_config config +end config :phoenix, :json_library, Jason @@ -60,4 +62,4 @@ config :logger, :error, # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. -import_config "#{Mix.env()}.exs" +import_config "#{config_env()}.exs" diff --git a/config/dev.exs b/config/dev.exs index 7f8759b576..21e6c9fd40 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config # DO NOT make it `:debug` or all Ecto logs will be shown for indexer config :logger, :console, level: :info diff --git a/config/prod.exs b/config/prod.exs index 5503ebbacf..2175810d04 100644 --- a/config/prod.exs +++ b/config/prod.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config # Do not print debug messages in production diff --git a/config/runtime.exs b/config/runtime.exs new file mode 100644 index 0000000000..8c662645e6 --- /dev/null +++ b/config/runtime.exs @@ -0,0 +1,15 @@ +import Config + +import Bitwise + +indexer_memory_limit = + "INDEXER_MEMORY_LIMIT" + |> System.get_env("1") + |> Integer.parse() + |> case do + {integer, ""} -> integer + _ -> 1 + end + +config :indexer, + memory_limit: indexer_memory_limit <<< 30 diff --git a/config/test.exs b/config/test.exs index ad8cdb57c4..915feb80e8 100644 --- a/config/test.exs +++ b/config/test.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config # Print only warnings and errors during test diff --git a/docker/Makefile b/docker/Makefile index c969d511db..a9439db24b 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -392,6 +392,9 @@ endif ifdef COIN_BALANCE_ON_DEMAND_FETCHER_THRESHOLD_MINUTES BLOCKSCOUT_CONTAINER_PARAMS += -e 'COIN_BALANCE_ON_DEMAND_FETCHER_THRESHOLD_MINUTES=$(COIN_BALANCE_ON_DEMAND_FETCHER_THRESHOLD_MINUTES)' endif +ifdef INDEXER_MEMORY_LIMIT + BLOCKSCOUT_CONTAINER_PARAMS += -e 'INDEXER_MEMORY_LIMIT=$(INDEXER_MEMORY_LIMIT)' +endif HAS_BLOCKSCOUT_IMAGE := $(shell docker images | grep -sw "${BS_CONTAINER_IMAGE} ") build: diff --git a/mix.exs b/mix.exs index 9f256f78dd..bbf55f4c99 100644 --- a/mix.exs +++ b/mix.exs @@ -6,7 +6,7 @@ defmodule BlockScout.Mixfile do def project do [ app: :block_scout, - aliases: aliases(Mix.env()), + # aliases: aliases(config_env()), version: "4.1.2", apps_path: "apps", deps: deps(), @@ -16,7 +16,7 @@ defmodule BlockScout.Mixfile do credo: :test, dialyzer: :test ], - start_permanent: Mix.env() == :prod, + # start_permanent: config_env() == :prod, releases: [ blockscout: [ applications: [ @@ -42,23 +42,23 @@ defmodule BlockScout.Mixfile do ] end - defp aliases(env) do - [ - # to match behavior of `mix test` in `apps/indexer`, which needs to not start applications for `indexer` to - # prevent its supervision tree from starting, which is undesirable in test - test: "test --no-start" - ] ++ env_aliases(env) - end + # defp aliases(env) do + # [ + # # to match behavior of `mix test` in `apps/indexer`, which needs to not start applications for `indexer` to + # # prevent its supervision tree from starting, which is undesirable in test + # test: "test --no-start" + # ] ++ env_aliases(env) + # end - defp env_aliases(:dev) do - [] - end + # defp env_aliases(:dev) do + # [] + # end - defp env_aliases(_env) do - [ - compile: "compile --warnings-as-errors" - ] - end + # defp env_aliases(_env) do + # [ + # compile: "compile --warnings-as-errors" + # ] + # end # Dependencies can be Hex packages: # diff --git a/rel/config.exs b/rel/config.exs index 8227d36621..eaaa52e36b 100644 --- a/rel/config.exs +++ b/rel/config.exs @@ -1,3 +1,5 @@ +import Config + # Import all plugins from `rel/plugins` # They can then be used by adding `plugin MyPlugin` to # either an environment, or release definition, where @@ -33,7 +35,7 @@ use Mix.Releases.Config, # This sets the default release built by `mix release` default_release: :default, # This sets the default environment used by `mix release` - default_environment: Mix.env() + default_environment: config_env() # For a full list of config options for both releases # and environments, visit https://hexdocs.pm/distillery/config/distillery.html