Merge pull request #5192 from blockscout/vb-db-config-parser

Fix DATABASE_URL config parser

CHANGELOG.md

@@ -5,6 +5,7 @@
 - [#4690](https://github.com/blockscout/blockscout/pull/4690) - Improve pagination: introduce pagination with random access to pages; Integrate it to the Transactions List page
 
 ### Fixes
+- [#5192](https://github.com/blockscout/blockscout/pull/5192) - Fix DATABASE_URL config parser
 - [#5169](https://github.com/blockscout/blockscout/pull/5169) - Fix several UI bugs; Add tooltip to the prev/next block buttons
 - [#5184](https://github.com/blockscout/blockscout/pull/5184) - eth_call method: remove from param from the request, if it is null
 - [#5172](https://github.com/blockscout/blockscout/pull/5172), [#5182](https://github.com/blockscout/blockscout/pull/5182) - Reduced the size of js bundles

apps/explorer/config/config.exs

@@ -29,7 +29,7 @@ config :explorer, Explorer.Counters.AverageBlockTime,
 
 config :explorer, Explorer.Chain.Events.Listener,
   enabled:
-    if(System.get_env("DISABLE_WEBAPP") == nil && System.get_env("DISABLE_INDEXER") == nil,
+    if(System.get_env("DISABLE_WEBAPP") == "true" && System.get_env("DISABLE_INDEXER") == "true",
       do: false,
       else: true
     )
@@ -140,6 +140,8 @@ config :explorer, Explorer.Integrations.EctoLogger, query_time_ms_threshold: :ti
 
 config :explorer, Explorer.Market.History.Cataloger, enabled: System.get_env("DISABLE_INDEXER") != "true"
 
+config :explorer, Explorer.Chain.Cache.MinMissingBlockNumber, enabled: System.get_env("DISABLE_WRITE_API") != "true"
+
 txs_stats_init_lag =
   System.get_env("TXS_HISTORIAN_INIT_LAG", "0")
   |> Integer.parse()

apps/explorer/lib/explorer/application.ex

@@ -63,8 +63,7 @@ defmodule Explorer.Application do
       con_cache_child_spec(RSK.cache_name(), ttl_check_interval: :timer.minutes(1), global_ttl: :timer.minutes(30)),
       Transactions,
       Accounts,
-      Uncles,
+      Uncles
-      MinMissingBlockNumber
     ]
 
     children = base_children ++ configurable_children()
@@ -96,7 +95,8 @@ defmodule Explorer.Application do
       configure(Explorer.Counters.AverageBlockTime),
       configure(Explorer.Counters.Bridge),
       configure(Explorer.Validator.MetadataProcessor),
-      configure(Explorer.Staking.ContractState)
+      configure(Explorer.Staking.ContractState),
+      configure(MinMissingBlockNumber)
     ]
     |> List.flatten()
   end

apps/explorer/lib/explorer/chain.ex

@@ -6672,30 +6672,6 @@ defmodule Explorer.Chain do
 
   defp boolean_to_check_result(false), do: :not_found
 
-  def extract_db_name(db_url) do
-    if db_url == nil do
-      ""
-    else
-      db_url
-      |> String.split("/")
-      |> Enum.take(-1)
-      |> Enum.at(0)
-    end
-  end
-
-  def extract_db_host(db_url) do
-    if db_url == nil do
-      ""
-    else
-      db_url
-      |> String.split("@")
-      |> Enum.take(-1)
-      |> Enum.at(0)
-      |> String.split(":")
-      |> Enum.at(0)
-    end
-  end
-
   @doc """
   Fetches the first trace from the Parity trace URL.
   """

apps/explorer/lib/explorer/chain/events/listener.ex

@@ -6,23 +6,15 @@ defmodule Explorer.Chain.Events.Listener do
   use GenServer
 
   alias Postgrex.Notifications
-  import Explorer.Chain, only: [extract_db_name: 1, extract_db_host: 1]
 
   def start_link(_) do
     GenServer.start_link(__MODULE__, "chain_event", name: __MODULE__)
   end
 
   def init(channel) do
-    explorer_repo =
+    {:ok, pid} =
       :explorer
       |> Application.get_env(Explorer.Repo)
-
-    db_url = explorer_repo[:url]
-
-    {:ok, pid} =
-      explorer_repo
-      |> Keyword.put(:database, extract_db_name(db_url))
-      |> Keyword.put(:hostname, extract_db_host(db_url))
       |> Notifications.start_link()
 
     ref = Notifications.listen!(pid, channel)

apps/explorer/lib/explorer/repo.ex

@@ -5,12 +5,28 @@ defmodule Explorer.Repo do
 
   require Logger
 
+  alias Explorer.Repo.ConfigHelper
+
   @doc """
   Dynamically loads the repository url from the
   DATABASE_URL environment variable.
   """
   def init(_, opts) do
-    {:ok, Keyword.put(opts, :url, System.get_env("DATABASE_URL"))}
+    db_url = System.get_env("DATABASE_URL")
+    repo_conf = Application.get_env(:explorer, Explorer.Repo)
+
+    merged =
+      %{url: db_url}
+      |> ConfigHelper.get_db_config()
+      |> Keyword.merge(repo_conf, fn
+        _key, v1, nil -> v1
+        _key, nil, v2 -> v2
+        _, _, v2 -> v2
+      end)
+
+    Application.put_env(:explorer, Explorer.Repo, merged)
+
+    {:ok, Keyword.put(opts, :url, db_url)}
   end
 
   def logged_transaction(fun_or_multi, opts \\ []) do

apps/explorer/lib/explorer/repo/config_helper.ex

@@ -0,0 +1,47 @@
+defmodule Explorer.Repo.ConfigHelper do
+  @moduledoc """
+  Extracts values from environment and adds them to application config.
+
+  Notably, this module processes the DATABASE_URL environment variable and extracts discrete parameters.
+
+  The priority of vars is postgrex enviroment vars < DATABASE_URL components, with values being overwritted by higher priority.
+  """
+
+  # https://hexdocs.pm/postgrex/Postgrex.html#start_link/1-options
+  @postgrex_env_vars [
+    username: "PGUSER",
+    password: "PGPASSWORD",
+    host: "PGHOST",
+    port: "PGPORT",
+    database: "PGDATABASE"
+  ]
+
+  def get_db_config(opts) do
+    url = opts[:url] || System.get_env("DATABASE_URL")
+    env_function = opts[:env_func] || (&System.get_env/1)
+
+    @postgrex_env_vars
+    |> get_env_vars(env_function)
+    |> Keyword.merge(extract_parameters(url))
+  end
+
+  defp extract_parameters(empty) when empty == nil or empty == "", do: []
+
+  # sobelow_skip ["DOS.StringToAtom"]
+  defp extract_parameters(database_url) do
+    ~r/\w*:\/\/(?<username>\w+):(?<password>\w*)?@(?<hostname>[a-zA-Z\d\.]+):(?<port>\d+)\/(?<database>\w+)/
+    |> Regex.named_captures(database_url)
+    |> Keyword.new(fn {k, v} -> {String.to_atom(k), v} end)
+    |> Keyword.put(:url, database_url)
+  end
+
+  defp get_env_vars(vars, env_function) do
+    Enum.reduce(vars, [], fn {name, var}, opts ->
+      case env_function.(var) do
+        nil -> opts
+        "" -> opts
+        env_value -> Keyword.put(opts, name, env_value)
+      end
+    end)
+  end
+end

apps/explorer/test/explorer/chain_test.exs

@@ -5334,40 +5334,6 @@ defmodule Explorer.ChainTest do
     end
   end
 
-  describe "extract_db_name/1" do
-    test "extracts correct db name" do
-      db_url = "postgresql://viktor:@localhost:5432/blockscout-dev-1"
-      assert Chain.extract_db_name(db_url) == "blockscout-dev-1"
-    end
-
-    test "returns empty db name" do
-      db_url = ""
-      assert Chain.extract_db_name(db_url) == ""
-    end
-
-    test "returns nil db name" do
-      db_url = nil
-      assert Chain.extract_db_name(db_url) == ""
-    end
-  end
-
-  describe "extract_db_host/1" do
-    test "extracts correct db host" do
-      db_url = "postgresql://viktor:@localhost:5432/blockscout-dev-1"
-      assert Chain.extract_db_host(db_url) == "localhost"
-    end
-
-    test "returns empty db name" do
-      db_url = ""
-      assert Chain.extract_db_host(db_url) == ""
-    end
-
-    test "returns nil db name" do
-      db_url = nil
-      assert Chain.extract_db_host(db_url) == ""
-    end
-  end
-
   describe "fetch_first_trace/2" do
     test "fetched first trace", %{
       json_rpc_named_arguments: json_rpc_named_arguments

apps/explorer/test/explorer/repo/config_helper_test.exs

@@ -0,0 +1,71 @@
+defmodule Explorer.Repo.ConfigHelperTest do
+  use Explorer.DataCase
+
+  alias Explorer.Repo.ConfigHelper
+
+  describe "get_db_config/1" do
+    test "parse params from database url" do
+      database_url = "postgresql://test_username:test_password@127.8.8.1:7777/test_database"
+
+      result = ConfigHelper.get_db_config(%{url: database_url, env_func: fn _ -> nil end})
+
+      assert result[:username] == "test_username"
+      assert result[:password] == "test_password"
+      assert result[:hostname] == "127.8.8.1"
+      assert result[:port] == "7777"
+      assert result[:database] == "test_database"
+    end
+
+    test "get username without password" do
+      database_url = "postgresql://test_username:@127.8.8.1:7777/test_database"
+
+      result = ConfigHelper.get_db_config(%{url: database_url, env_func: fn _ -> nil end})
+
+      assert result[:username] == "test_username"
+      assert result[:password] == ""
+      assert result[:hostname] == "127.8.8.1"
+      assert result[:port] == "7777"
+      assert result[:database] == "test_database"
+    end
+
+    test "get hostname instead of ip" do
+      database_url = "postgresql://test_username:@cooltesthost:7777/test_database"
+
+      result = ConfigHelper.get_db_config(%{url: database_url, env_func: fn _ -> nil end})
+
+      assert result[:username] == "test_username"
+      assert result[:password] == ""
+      assert result[:hostname] == "cooltesthost"
+      assert result[:port] == "7777"
+      assert result[:database] == "test_database"
+    end
+
+    test "overwrite postgrex vars param with database url" do
+      database_url = "postgresql://test_username:@127.8.8.1:7777/test_database"
+
+      vars = %{"PGUSER" => "postgrex_user", "PGPASSWORD" => "postgrex_password"}
+      func = fn v -> vars[v] end
+      result = ConfigHelper.get_db_config(%{url: database_url, env_func: func})
+
+      assert result[:username] == "test_username"
+      assert result[:password] == ""
+      assert result[:hostname] == "127.8.8.1"
+      assert result[:port] == "7777"
+      assert result[:database] == "test_database"
+    end
+
+    test "overwrite database password param with empty DATABASE_URL password" do
+      database_url = "postgresql://test_username:@127.8.8.1:7777/test_database"
+
+      vars = %{"PGUSER" => "postgrex_user", "PGPASSWORD" => "postgrex_password"}
+      func = fn v -> vars[v] end
+      result = ConfigHelper.get_db_config(%{url: database_url, env_func: func})
+
+      assert result[:username] == "test_username"
+      assert result[:password] == ""
+      assert result[:hostname] == "127.8.8.1"
+      assert result[:port] == "7777"
+      assert result[:database] == "test_database"
+    end
+  end
+end