Extract database parameters from DATABASE_URL. (#305)

* Extract database parameters from DATABASE_URL.

apps/explorer/lib/explorer/chain.ex

@@ -6672,30 +6672,6 @@ defmodule Explorer.Chain do
 
   defp boolean_to_check_result(false), do: :not_found
 
-  def extract_db_name(db_url) do
-    if db_url == nil do
-      ""
-    else
-      db_url
-      |> String.split("/")
-      |> Enum.take(-1)
-      |> Enum.at(0)
-    end
-  end
-
-  def extract_db_host(db_url) do
-    if db_url == nil do
-      ""
-    else
-      db_url
-      |> String.split("@")
-      |> Enum.take(-1)
-      |> Enum.at(0)
-      |> String.split(":")
-      |> Enum.at(0)
-    end
-  end
-
   @doc """
   Fetches the first trace from the Parity trace URL.
   """

apps/explorer/lib/explorer/chain/events/listener.ex

@@ -6,23 +6,15 @@ defmodule Explorer.Chain.Events.Listener do
   use GenServer
 
   alias Postgrex.Notifications
-  import Explorer.Chain, only: [extract_db_name: 1, extract_db_host: 1]
 
   def start_link(_) do
     GenServer.start_link(__MODULE__, "chain_event", name: __MODULE__)
   end
 
   def init(channel) do
-    explorer_repo =
+    {:ok, pid} =
       :explorer
       |> Application.get_env(Explorer.Repo)
-
-    db_url = explorer_repo[:url]
-
-    {:ok, pid} =
-      explorer_repo
-      |> Keyword.put(:database, extract_db_name(db_url))
-      |> Keyword.put(:hostname, extract_db_host(db_url))
       |> Notifications.start_link()
 
     ref = Notifications.listen!(pid, channel)

apps/explorer/lib/explorer/repo.ex

@@ -5,12 +5,28 @@ defmodule Explorer.Repo do
 
   require Logger
 
+  alias Explorer.Repo.ConfigHelper
+
   @doc """
   Dynamically loads the repository url from the
   DATABASE_URL environment variable.
   """
   def init(_, opts) do
-    {:ok, Keyword.put(opts, :url, System.get_env("DATABASE_URL"))}
+    db_url = System.get_env("DATABASE_URL")
+    repo_conf = Application.get_env(:explorer, Explorer.Repo)
+
+    merged =
+      %{url: db_url}
+      |> ConfigHelper.get_db_config()
+      |> Keyword.merge(repo_conf, fn
+        _key, v1, nil -> v1
+        _key, nil, v2 -> v2
+        _, _, v2 -> v2
+      end)
+
+    Application.put_env(:explorer, Explorer.Repo, merged)
+
+    {:ok, Keyword.put(opts, :url, db_url)}
   end
 
   def logged_transaction(fun_or_multi, opts \\ []) do

apps/explorer/lib/explorer/repo/config_helper.ex

@@ -0,0 +1,62 @@
+defmodule Explorer.Repo.ConfigHelper do
+  @moduledoc """
+  Extracts values from environment and adds them to application config.
+
+  Notably, this module processes the DATABASE_URL environment variable and extracts discrete parameters.
+
+  The priority of vars is DATABASE_URL components < postgrex enviroment vars < application config vars, with values being overwritted by higher priority.
+  """
+
+  # set in apps/*/config/*.exs
+  @app_env_vars [
+    username: "DATABASE_USER",
+    password: "DATABASE_PASSWORD",
+    host: "DATABASE_HOST",
+    port: "DATABASE_PORT",
+    database: "DATABASE_DB"
+  ]
+
+  # https://hexdocs.pm/postgrex/Postgrex.html#start_link/1-options
+  @postgrex_env_vars [
+    username: "PGUSER",
+    password: "PGPASSWORD",
+    host: "PGHOST",
+    port: "PGPORT",
+    database: "PGDATABASE"
+  ]
+
+  def get_db_config(opts) do
+    url = opts[:url] || System.get_env("DATABASE_URL")
+    env_function = opts[:env_func] || (&System.get_env/1)
+
+    url
+    |> extract_parameters()
+    |> Keyword.merge(get_env_vars(@postgrex_env_vars, env_function))
+    |> Keyword.merge(get_env_vars(@app_env_vars, env_function))
+  end
+
+  defp extract_parameters(empty) when empty == nil or empty == "", do: []
+
+  # sobelow_skip ["DOS.StringToAtom"]
+  defp extract_parameters(database_url) do
+    ~r/\w*:\/\/(?<username>\w+):(?<password>\w*)?@(?<hostname>[a-zA-Z\d\.]+):(?<port>\d+)\/(?<database>\w+)/
+    |> Regex.named_captures(database_url)
+    |> Keyword.new(fn {k, v} -> {String.to_atom(k), v} end)
+    |> Keyword.put(:url, database_url)
+    |> Enum.filter(fn
+      # don't include keys with empty values
+      {_, ""} -> false
+      _ -> true
+    end)
+  end
+
+  defp get_env_vars(vars, env_function) do
+    Enum.reduce(vars, [], fn {name, var}, opts ->
+      case env_function.(var) do
+        nil -> opts
+        "" -> opts
+        env_value -> Keyword.put(opts, name, env_value)
+      end
+    end)
+  end
+end

apps/explorer/test/explorer/chain_test.exs

@@ -5334,40 +5334,6 @@ defmodule Explorer.ChainTest do
     end
   end
 
-  describe "extract_db_name/1" do
-    test "extracts correct db name" do
-      db_url = "postgresql://viktor:@localhost:5432/blockscout-dev-1"
-      assert Chain.extract_db_name(db_url) == "blockscout-dev-1"
-    end
-
-    test "returns empty db name" do
-      db_url = ""
-      assert Chain.extract_db_name(db_url) == ""
-    end
-
-    test "returns nil db name" do
-      db_url = nil
-      assert Chain.extract_db_name(db_url) == ""
-    end
-  end
-
-  describe "extract_db_host/1" do
-    test "extracts correct db host" do
-      db_url = "postgresql://viktor:@localhost:5432/blockscout-dev-1"
-      assert Chain.extract_db_host(db_url) == "localhost"
-    end
-
-    test "returns empty db name" do
-      db_url = ""
-      assert Chain.extract_db_host(db_url) == ""
-    end
-
-    test "returns nil db name" do
-      db_url = nil
-      assert Chain.extract_db_host(db_url) == ""
-    end
-  end
-
   describe "fetch_first_trace/2" do
     test "fetched first trace", %{
       json_rpc_named_arguments: json_rpc_named_arguments

apps/explorer/test/explorer/repo/config_helper_test.exs

@@ -0,0 +1,72 @@
+defmodule Explorer.Repo.ConfigHelperTest do
+  use Explorer.DataCase
+
+  alias Explorer.Repo.ConfigHelper
+
+  describe "get_db_config/1" do
+    test "parse params from database url" do
+      database_url = "postgresql://test_username:test_password@127.8.8.1:7777/test_database"
+
+      result = ConfigHelper.get_db_config(%{url: database_url, env_func: fn _ -> nil end})
+
+      assert result[:username] == "test_username"
+      assert result[:password] == "test_password"
+      assert result[:hostname] == "127.8.8.1"
+      assert result[:port] == "7777"
+      assert result[:database] == "test_database"
+    end
+
+    test "get username without password" do
+      database_url = "postgresql://test_username:@127.8.8.1:7777/test_database"
+
+      result = ConfigHelper.get_db_config(%{url: database_url, env_func: fn _ -> nil end})
+
+      assert result[:username] == "test_username"
+      refute result[:password]
+      assert result[:hostname] == "127.8.8.1"
+      assert result[:port] == "7777"
+      assert result[:database] == "test_database"
+    end
+
+    test "get hostname instead of ip" do
+      database_url = "postgresql://test_username:@cooltesthost:7777/test_database"
+
+      result = ConfigHelper.get_db_config(%{url: database_url, env_func: fn _ -> nil end})
+
+      assert result[:username] == "test_username"
+      refute result[:password]
+      assert result[:hostname] == "cooltesthost"
+      assert result[:port] == "7777"
+      assert result[:database] == "test_database"
+    end
+
+    test "overwrite database url param with postgrex vars" do
+      database_url = "postgresql://test_username:@127.8.8.1:7777/test_database"
+
+      vars = %{"PGUSER" => "postgrex_user", "PGPASSWORD" => "postgrex_password"}
+      func = fn v -> vars[v] end
+      result = ConfigHelper.get_db_config(%{url: database_url, env_func: func})
+
+      assert result[:username] == "postgrex_user"
+      assert result[:password] == "postgrex_password"
+      assert result[:hostname] == "127.8.8.1"
+      assert result[:port] == "7777"
+      assert result[:database] == "test_database"
+    end
+
+    test "overwrite database url param and postgrex with app env" do
+      database_url = "postgresql://test_username:@127.8.8.1:7777/test_database"
+
+      vars = %{"PGUSER" => "postgrex_user", "PGPASSWORD" => "postgrex_password", "DATABASE_USER" => "app_db_user"}
+      func = fn v -> vars[v] end
+
+      result = ConfigHelper.get_db_config(%{url: database_url, env_func: func})
+
+      assert result[:username] == "app_db_user"
+      assert result[:password] == "postgrex_password"
+      assert result[:hostname] == "127.8.8.1"
+      assert result[:port] == "7777"
+      assert result[:database] == "test_database"
+    end
+  end
+end