From c18b929b9eead5172511f959c3c234d037ff7740 Mon Sep 17 00:00:00 2001 From: Viktor Baranov Date: Mon, 23 Oct 2023 21:45:06 +0300 Subject: [PATCH] Fix db pool size exceeds Postgres max connections --- CHANGELOG.md | 2 + apps/explorer/lib/explorer/application.ex | 105 +++++++++++++--------- config/runtime/dev.exs | 8 +- config/runtime/prod.exs | 20 ++--- 4 files changed, 80 insertions(+), 55 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 25082d68b2..2d4a83a229 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ ### Fixes +- [#8685](https://github.com/blockscout/blockscout/pull/8685) - Fix db pool size exceeds Postgres max connections + ### Chore
diff --git a/apps/explorer/lib/explorer/application.ex b/apps/explorer/lib/explorer/application.ex index c80c669229..adc04a9d0d 100644 --- a/apps/explorer/lib/explorer/application.ex +++ b/apps/explorer/lib/explorer/application.ex @@ -46,10 +46,6 @@ defmodule Explorer.Application do # Children to start in all environments base_children = [ Explorer.Repo, - Explorer.Repo.Replica1, - Explorer.Repo.Account, - Explorer.Repo.PolygonEdge, - Explorer.Repo.RSK, Explorer.Vault, Supervisor.child_spec({SpandexDatadog.ApiServer, datadog_opts()}, id: SpandexDatadog.ApiServer), Supervisor.child_spec({Task.Supervisor, name: Explorer.HistoryTaskSupervisor}, id: Explorer.HistoryTaskSupervisor), @@ -93,43 +89,70 @@ defmodule Explorer.Application do end defp configurable_children do - [ - configure(Explorer.ExchangeRates), - configure(Explorer.ExchangeRates.TokenExchangeRates), - configure(Explorer.ChainSpec.GenesisData), - configure(Explorer.Market.History.Cataloger), - configure(Explorer.Chain.Cache.ContractsCounter), - configure(Explorer.Chain.Cache.NewContractsCounter), - configure(Explorer.Chain.Cache.VerifiedContractsCounter), - configure(Explorer.Chain.Cache.NewVerifiedContractsCounter), - configure(Explorer.Chain.Cache.TransactionActionTokensData), - configure(Explorer.Chain.Cache.TransactionActionUniswapPools), - configure(Explorer.Chain.Cache.WithdrawalsSum), - configure(Explorer.Chain.Transaction.History.Historian), - configure(Explorer.Chain.Events.Listener), - configure(Explorer.Counters.AddressesWithBalanceCounter), - configure(Explorer.Counters.AddressesCounter), - configure(Explorer.Counters.AddressTransactionsCounter), - configure(Explorer.Counters.AddressTokenTransfersCounter), - configure(Explorer.Counters.AddressTransactionsGasUsageCounter), - configure(Explorer.Counters.AddressTokenUsdSum), - configure(Explorer.Counters.TokenHoldersCounter), - configure(Explorer.Counters.TokenTransfersCounter), - configure(Explorer.Counters.BlockBurnedFeeCounter), - configure(Explorer.Counters.BlockPriorityFeeCounter), - configure(Explorer.Counters.AverageBlockTime), - configure(Explorer.Counters.Bridge), - configure(Explorer.Validator.MetadataProcessor), - configure(Explorer.Tags.AddressTag.Cataloger), - configure(MinMissingBlockNumber), - configure(TokenTransferTokenIdMigration.Supervisor), - configure(Explorer.Chain.Fetcher.CheckBytecodeMatchingOnDemand), - configure(Explorer.Chain.Fetcher.FetchValidatorInfoOnDemand), - configure(Explorer.TokenInstanceOwnerAddressMigration.Supervisor), - sc_microservice_configure(Explorer.Chain.Fetcher.LookUpSmartContractSourcesOnDemand), - configure(Explorer.Chain.Cache.RootstockLockedBTC) - ] - |> List.flatten() + configurable_children_set = + [ + configure(Explorer.ExchangeRates), + configure(Explorer.ExchangeRates.TokenExchangeRates), + configure(Explorer.ChainSpec.GenesisData), + configure(Explorer.Market.History.Cataloger), + configure(Explorer.Chain.Cache.ContractsCounter), + configure(Explorer.Chain.Cache.NewContractsCounter), + configure(Explorer.Chain.Cache.VerifiedContractsCounter), + configure(Explorer.Chain.Cache.NewVerifiedContractsCounter), + configure(Explorer.Chain.Cache.TransactionActionTokensData), + configure(Explorer.Chain.Cache.TransactionActionUniswapPools), + configure(Explorer.Chain.Cache.WithdrawalsSum), + configure(Explorer.Chain.Transaction.History.Historian), + configure(Explorer.Chain.Events.Listener), + configure(Explorer.Counters.AddressesWithBalanceCounter), + configure(Explorer.Counters.AddressesCounter), + configure(Explorer.Counters.AddressTransactionsCounter), + configure(Explorer.Counters.AddressTokenTransfersCounter), + configure(Explorer.Counters.AddressTransactionsGasUsageCounter), + configure(Explorer.Counters.AddressTokenUsdSum), + configure(Explorer.Counters.TokenHoldersCounter), + configure(Explorer.Counters.TokenTransfersCounter), + configure(Explorer.Counters.BlockBurnedFeeCounter), + configure(Explorer.Counters.BlockPriorityFeeCounter), + configure(Explorer.Counters.AverageBlockTime), + configure(Explorer.Counters.Bridge), + configure(Explorer.Validator.MetadataProcessor), + configure(Explorer.Tags.AddressTag.Cataloger), + configure(MinMissingBlockNumber), + configure(TokenTransferTokenIdMigration.Supervisor), + configure(Explorer.Chain.Fetcher.CheckBytecodeMatchingOnDemand), + configure(Explorer.Chain.Fetcher.FetchValidatorInfoOnDemand), + configure(Explorer.TokenInstanceOwnerAddressMigration.Supervisor), + sc_microservice_configure(Explorer.Chain.Fetcher.LookUpSmartContractSourcesOnDemand), + configure(Explorer.Chain.Cache.RootstockLockedBTC) + ] + |> List.flatten() + + repos_by_chain_type() ++ account_repo() ++ replica_repo() ++ configurable_children_set + end + + defp repos_by_chain_type do + if Mix.env() == :test do + [Explorer.Repo.PolygonEdge, Explorer.Repo.RSK] + else + [] + end + end + + defp account_repo do + if System.get_env("ACCOUNT_DATABASE_URL") || Mix.env() == :test do + [Explorer.Repo.Account] + else + [] + end + end + + defp replica_repo do + if System.get_env("DATABASE_READ_ONLY_API_URL") do + [Explorer.Repo.Replica1] + else + [] + end end defp should_start?(process) do diff --git a/config/runtime/dev.exs b/config/runtime/dev.exs index 567d38d793..09a67485fa 100644 --- a/config/runtime/dev.exs +++ b/config/runtime/dev.exs @@ -74,14 +74,18 @@ config :explorer, Explorer.Repo.PolygonEdge, database: database, hostname: hostname, url: System.get_env("DATABASE_URL"), - pool_size: ConfigHelper.parse_integer_env_var("POLYGON_EDGE_POOL_SIZE", 10) + # actually this repo is not started, and its pool size remains unused. + # separating repos for different CHAIN_TYPE is implemented only for the sake of keeping DB schema update relevant to the current chain type + pool_size: 1 # Configure Rootstock database config :explorer, Explorer.Repo.RSK, database: database, hostname: hostname, url: System.get_env("DATABASE_URL"), - pool_size: pool_size + # actually this repo is not started, and its pool size remains unused. + # separating repos for different CHAIN_TYPE is implemented only for the sake of keeping DB schema update relevant to the current chain type + pool_size: 1 variant = Variant.get() diff --git a/config/runtime/prod.exs b/config/runtime/prod.exs index cbbc1af52b..a2acffc385 100644 --- a/config/runtime/prod.exs +++ b/config/runtime/prod.exs @@ -27,10 +27,7 @@ config :block_scout_web, BlockScoutWeb.Endpoint, ### Explorer ### ################ -pool_size = - if System.get_env("DATABASE_READ_ONLY_API_URL"), - do: ConfigHelper.parse_integer_env_var("POOL_SIZE", 50), - else: ConfigHelper.parse_integer_env_var("POOL_SIZE", 40) +pool_size = ConfigHelper.parse_integer_env_var("POOL_SIZE", 50) # Configures the database config :explorer, Explorer.Repo, @@ -38,15 +35,10 @@ config :explorer, Explorer.Repo, pool_size: pool_size, ssl: ExplorerConfigHelper.ssl_enabled?() -pool_size_api = - if System.get_env("DATABASE_READ_ONLY_API_URL"), - do: ConfigHelper.parse_integer_env_var("POOL_SIZE_API", 50), - else: ConfigHelper.parse_integer_env_var("POOL_SIZE_API", 10) - # Configures API the database config :explorer, Explorer.Repo.Replica1, url: ExplorerConfigHelper.get_api_db_url(), - pool_size: pool_size_api, + pool_size: ConfigHelper.parse_integer_env_var("POOL_SIZE_API", 50), ssl: ExplorerConfigHelper.ssl_enabled?() # Configures Account database @@ -58,13 +50,17 @@ config :explorer, Explorer.Repo.Account, # Configures PolygonEdge database config :explorer, Explorer.Repo.PolygonEdge, url: System.get_env("DATABASE_URL"), - pool_size: ConfigHelper.parse_integer_env_var("POLYGON_EDGE_POOL_SIZE", 50), + # actually this repo is not started, and its pool size remains unused. + # separating repos for different CHAIN_TYPE is implemented only for the sake of keeping DB schema update relevant to the current chain type + pool_size: 1, ssl: ExplorerConfigHelper.ssl_enabled?() # Configures Rootstock database config :explorer, Explorer.Repo.RSK, url: System.get_env("DATABASE_URL"), - pool_size: pool_size, + # actually this repo is not started, and its pool size remains unused. + # separating repos for different CHAIN_TYPE is implemented only for the sake of keeping DB schema update relevant to the current chain type + pool_size: 1, ssl: ExplorerConfigHelper.ssl_enabled?() variant = Variant.get()