Fix db pool size exceeds Postgres max connections

1 year ago · c18b929b9e
parent fd020a49ea
commit c18b929b9e
4 changed files with 80 additions and 55 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -6,6 +6,8 @@

 ### Fixes

+- [#8685](https://github.com/blockscout/blockscout/pull/8685) - Fix db pool size exceeds Postgres max connections
+
 ### Chore

 <details>
--- a/apps/explorer/lib/explorer/application.ex
+++ b/apps/explorer/lib/explorer/application.ex
@ -46,10 +46,6 @@ defmodule Explorer.Application do
    # Children to start in all environments
    base_children = [
      Explorer.Repo,
-      Explorer.Repo.Replica1,
-      Explorer.Repo.Account,
-      Explorer.Repo.PolygonEdge,
-      Explorer.Repo.RSK,
      Explorer.Vault,
      Supervisor.child_spec({SpandexDatadog.ApiServer, datadog_opts()}, id: SpandexDatadog.ApiServer),
      Supervisor.child_spec({Task.Supervisor, name: Explorer.HistoryTaskSupervisor}, id: Explorer.HistoryTaskSupervisor),
@ -93,43 +89,70 @@ defmodule Explorer.Application do
  end

  defp configurable_children do
-    [
-      configure(Explorer.ExchangeRates),
-      configure(Explorer.ExchangeRates.TokenExchangeRates),
-      configure(Explorer.ChainSpec.GenesisData),
-      configure(Explorer.Market.History.Cataloger),
-      configure(Explorer.Chain.Cache.ContractsCounter),
-      configure(Explorer.Chain.Cache.NewContractsCounter),
-      configure(Explorer.Chain.Cache.VerifiedContractsCounter),
-      configure(Explorer.Chain.Cache.NewVerifiedContractsCounter),
-      configure(Explorer.Chain.Cache.TransactionActionTokensData),
-      configure(Explorer.Chain.Cache.TransactionActionUniswapPools),
-      configure(Explorer.Chain.Cache.WithdrawalsSum),
-      configure(Explorer.Chain.Transaction.History.Historian),
-      configure(Explorer.Chain.Events.Listener),
-      configure(Explorer.Counters.AddressesWithBalanceCounter),
-      configure(Explorer.Counters.AddressesCounter),
-      configure(Explorer.Counters.AddressTransactionsCounter),
-      configure(Explorer.Counters.AddressTokenTransfersCounter),
-      configure(Explorer.Counters.AddressTransactionsGasUsageCounter),
-      configure(Explorer.Counters.AddressTokenUsdSum),
-      configure(Explorer.Counters.TokenHoldersCounter),
-      configure(Explorer.Counters.TokenTransfersCounter),
-      configure(Explorer.Counters.BlockBurnedFeeCounter),
-      configure(Explorer.Counters.BlockPriorityFeeCounter),
-      configure(Explorer.Counters.AverageBlockTime),
-      configure(Explorer.Counters.Bridge),
-      configure(Explorer.Validator.MetadataProcessor),
-      configure(Explorer.Tags.AddressTag.Cataloger),
-      configure(MinMissingBlockNumber),
-      configure(TokenTransferTokenIdMigration.Supervisor),
-      configure(Explorer.Chain.Fetcher.CheckBytecodeMatchingOnDemand),
-      configure(Explorer.Chain.Fetcher.FetchValidatorInfoOnDemand),
-      configure(Explorer.TokenInstanceOwnerAddressMigration.Supervisor),
-      sc_microservice_configure(Explorer.Chain.Fetcher.LookUpSmartContractSourcesOnDemand),
-      configure(Explorer.Chain.Cache.RootstockLockedBTC)
-    ]
-    |> List.flatten()
+    configurable_children_set =
+      [
+        configure(Explorer.ExchangeRates),
+        configure(Explorer.ExchangeRates.TokenExchangeRates),
+        configure(Explorer.ChainSpec.GenesisData),
+        configure(Explorer.Market.History.Cataloger),
+        configure(Explorer.Chain.Cache.ContractsCounter),
+        configure(Explorer.Chain.Cache.NewContractsCounter),
+        configure(Explorer.Chain.Cache.VerifiedContractsCounter),
+        configure(Explorer.Chain.Cache.NewVerifiedContractsCounter),
+        configure(Explorer.Chain.Cache.TransactionActionTokensData),
+        configure(Explorer.Chain.Cache.TransactionActionUniswapPools),
+        configure(Explorer.Chain.Cache.WithdrawalsSum),
+        configure(Explorer.Chain.Transaction.History.Historian),
+        configure(Explorer.Chain.Events.Listener),
+        configure(Explorer.Counters.AddressesWithBalanceCounter),
+        configure(Explorer.Counters.AddressesCounter),
+        configure(Explorer.Counters.AddressTransactionsCounter),
+        configure(Explorer.Counters.AddressTokenTransfersCounter),
+        configure(Explorer.Counters.AddressTransactionsGasUsageCounter),
+        configure(Explorer.Counters.AddressTokenUsdSum),
+        configure(Explorer.Counters.TokenHoldersCounter),
+        configure(Explorer.Counters.TokenTransfersCounter),
+        configure(Explorer.Counters.BlockBurnedFeeCounter),
+        configure(Explorer.Counters.BlockPriorityFeeCounter),
+        configure(Explorer.Counters.AverageBlockTime),
+        configure(Explorer.Counters.Bridge),
+        configure(Explorer.Validator.MetadataProcessor),
+        configure(Explorer.Tags.AddressTag.Cataloger),
+        configure(MinMissingBlockNumber),
+        configure(TokenTransferTokenIdMigration.Supervisor),
+        configure(Explorer.Chain.Fetcher.CheckBytecodeMatchingOnDemand),
+        configure(Explorer.Chain.Fetcher.FetchValidatorInfoOnDemand),
+        configure(Explorer.TokenInstanceOwnerAddressMigration.Supervisor),
+        sc_microservice_configure(Explorer.Chain.Fetcher.LookUpSmartContractSourcesOnDemand),
+        configure(Explorer.Chain.Cache.RootstockLockedBTC)
+      ]
+      |> List.flatten()
+
+    repos_by_chain_type() ++ account_repo() ++ replica_repo() ++ configurable_children_set
+  end
+
+  defp repos_by_chain_type do
+    if Mix.env() == :test do
+      [Explorer.Repo.PolygonEdge, Explorer.Repo.RSK]
+    else
+      []
+    end
+  end
+
+  defp account_repo do
+    if System.get_env("ACCOUNT_DATABASE_URL") || Mix.env() == :test do
+      [Explorer.Repo.Account]
+    else
+      []
+    end
+  end
+
+  defp replica_repo do
+    if System.get_env("DATABASE_READ_ONLY_API_URL") do
+      [Explorer.Repo.Replica1]
+    else
+      []
+    end
  end

  defp should_start?(process) do
--- a/config/runtime/dev.exs
+++ b/config/runtime/dev.exs
@ -74,14 +74,18 @@ config :explorer, Explorer.Repo.PolygonEdge,
  database: database,
  hostname: hostname,
  url: System.get_env("DATABASE_URL"),
-  pool_size: ConfigHelper.parse_integer_env_var("POLYGON_EDGE_POOL_SIZE", 10)
+  # actually this repo is not started, and its pool size remains unused.
+  # separating repos for different CHAIN_TYPE is implemented only for the sake of keeping DB schema update relevant to the current chain type
+  pool_size: 1

 # Configure Rootstock database
 config :explorer, Explorer.Repo.RSK,
  database: database,
  hostname: hostname,
  url: System.get_env("DATABASE_URL"),
-  pool_size: pool_size
+  # actually this repo is not started, and its pool size remains unused.
+  # separating repos for different CHAIN_TYPE is implemented only for the sake of keeping DB schema update relevant to the current chain type
+  pool_size: 1

 variant = Variant.get()

--- a/config/runtime/prod.exs
+++ b/config/runtime/prod.exs
@ -27,10 +27,7 @@ config :block_scout_web, BlockScoutWeb.Endpoint,
 ### Explorer ###
 ################

-pool_size =
-  if System.get_env("DATABASE_READ_ONLY_API_URL"),
-    do: ConfigHelper.parse_integer_env_var("POOL_SIZE", 50),
-    else: ConfigHelper.parse_integer_env_var("POOL_SIZE", 40)
+pool_size = ConfigHelper.parse_integer_env_var("POOL_SIZE", 50)

 # Configures the database
 config :explorer, Explorer.Repo,
@ -38,15 +35,10 @@ config :explorer, Explorer.Repo,
  pool_size: pool_size,
  ssl: ExplorerConfigHelper.ssl_enabled?()

-pool_size_api =
-  if System.get_env("DATABASE_READ_ONLY_API_URL"),
-    do: ConfigHelper.parse_integer_env_var("POOL_SIZE_API", 50),
-    else: ConfigHelper.parse_integer_env_var("POOL_SIZE_API", 10)
-
 # Configures API the database
 config :explorer, Explorer.Repo.Replica1,
  url: ExplorerConfigHelper.get_api_db_url(),
-  pool_size: pool_size_api,
+  pool_size: ConfigHelper.parse_integer_env_var("POOL_SIZE_API", 50),
  ssl: ExplorerConfigHelper.ssl_enabled?()

 # Configures Account database
@ -58,13 +50,17 @@ config :explorer, Explorer.Repo.Account,
 # Configures PolygonEdge database
 config :explorer, Explorer.Repo.PolygonEdge,
  url: System.get_env("DATABASE_URL"),
-  pool_size: ConfigHelper.parse_integer_env_var("POLYGON_EDGE_POOL_SIZE", 50),
+  # actually this repo is not started, and its pool size remains unused.
+  # separating repos for different CHAIN_TYPE is implemented only for the sake of keeping DB schema update relevant to the current chain type
+  pool_size: 1,
  ssl: ExplorerConfigHelper.ssl_enabled?()

 # Configures Rootstock database
 config :explorer, Explorer.Repo.RSK,
  url: System.get_env("DATABASE_URL"),
-  pool_size: pool_size,
+  # actually this repo is not started, and its pool size remains unused.
+  # separating repos for different CHAIN_TYPE is implemented only for the sake of keeping DB schema update relevant to the current chain type
+  pool_size: 1,
  ssl: ExplorerConfigHelper.ssl_enabled?()

 variant = Variant.get()