Re-enable Ecto loggers

Allow Ecto loggers to log to the application-specific log **file**,
`log/<Mix.env>/ecto.log` but raise `:console`'s level to `:info`, so that Ecto
output does not flood `iex` and make it unusable.

README.md

@@ -85,7 +85,7 @@ _Additional runtime options:_
 `iex -S mix phx.server`
 
 *  Run Phoenix Server with real time indexer
-`DEBUG_INDEXER=1 iex -S mix phx.server`
+`iex -S mix phx.server`
 
 ### BlockScout Visual Interface
 

apps/block_scout_web/config/config.exs

@@ -35,7 +35,7 @@ config :ex_cldr,
 config :logger, :block_scout_web,
   # keep synced with `config/config.exs`
   format: "$time $metadata[$level] $message\n",
-  metadata: [:request_id],
+  metadata: [:application, :request_id],
   metadata_filter: [application: :block_scout_web]
 
 # Import environment specific config. This must remain at the bottom

apps/ethereum_jsonrpc/config/config.exs

@@ -3,7 +3,7 @@ use Mix.Config
 config :logger, :ethereum_jsonrpc,
   # keep synced with `config/config.exs`
   format: "$time $metadata[$level] $message\n",
-  metadata: [:request_id],
+  metadata: [:application, :request_id],
   metadata_filter: [application: :ethereum_jsonrpc]
 
 # Import environment specific config. This must remain at the bottom

apps/explorer/config/config.exs

@@ -26,7 +26,7 @@ config :explorer,
 config :logger, :explorer,
   # keep synced with `config/config.exs`
   format: "$time $metadata[$level] $message\n",
-  metadata: [:request_id],
+  metadata: [:application, :request_id],
   metadata_filter: [application: :explorer]
 
 # Import environment specific config. This must remain at the bottom

apps/explorer/config/dev.exs

@@ -5,7 +5,6 @@ config :explorer, Explorer.Repo,
   adapter: Ecto.Adapters.Postgres,
   database: "explorer_dev",
   hostname: "localhost",
-  loggers: [],
   pool_size: 20,
   pool_timeout: 60_000,
   timeout: 80_000

apps/indexer/config/config.exs

@@ -3,13 +3,12 @@
 use Mix.Config
 
 config :indexer,
-  debug_logs: !!System.get_env("DEBUG_INDEXER"),
   ecto_repos: [Explorer.Repo]
 
 config :logger, :indexer,
   # keep synced with `config/config.exs`
   format: "$time $metadata[$level] $message\n",
-  metadata: [:request_id],
+  metadata: [:application, :request_id],
   metadata_filter: [application: :indexer]
 
 # Import environment specific config. This must remain at the bottom

apps/indexer/lib/indexer.ex

@@ -2,7 +2,6 @@ defmodule Indexer do
   @moduledoc """
   Indexes an Ethereum-based chain using JSONRPC.
   """
-  require Logger
 
   alias Explorer.Chain
 
@@ -53,33 +52,4 @@ defmodule Indexer do
       num -> num + 1
     end
   end
-
-  @doc """
-  Logs debug message if `:debug_logs` have been enabled.
-  """
-  def debug(message_func) when is_function(message_func, 0) do
-    if debug_logs_enabled?() do
-      Logger.debug(message_func)
-    else
-      :noop
-    end
-  end
-
-  @doc """
-  Enables debug logs for indexing system.
-  """
-  def enable_debug_logs do
-    Application.put_env(:indexer, :debug_logs, true)
-  end
-
-  @doc """
-  Disables debug logs for indexing system.
-  """
-  def disable_debug_logs do
-    Application.put_env(:indexer, :debug_logs, false)
-  end
-
-  defp debug_logs_enabled? do
-    Application.fetch_env!(:indexer, :debug_logs)
-  end
 end

apps/indexer/lib/indexer/balance_fetcher.ex

@@ -4,6 +4,8 @@ defmodule Indexer.BalanceFetcher do
   `fetched_balance_block_number` to value at max `t:Explorer.Chain.Balance.t/0` `block_number` for the given `t:Explorer.Chain.Address.t/` `hash`.
   """
 
+  require Logger
+
   import EthereumJSONRPC, only: [integer_to_quantity: 1]
 
   alias Explorer.Chain
@@ -68,7 +70,7 @@ defmodule Indexer.BalanceFetcher do
     # `{address, block}`, so take unique params only
     unique_params_list = Enum.uniq(params_list)
 
-    Indexer.debug(fn -> "fetching #{length(unique_params_list)} balances" end)
+    Logger.debug(fn -> "fetching #{length(unique_params_list)} balances" end)
 
     case EthereumJSONRPC.fetch_balances(unique_params_list, json_rpc_named_arguments) do
       {:ok, balances_params} ->
@@ -87,7 +89,7 @@ defmodule Indexer.BalanceFetcher do
         :ok
 
       {:error, reason} ->
-        Indexer.debug(fn -> "failed to fetch #{length(unique_params_list)} balances, #{inspect(reason)}" end)
+        Logger.debug(fn -> "failed to fetch #{length(unique_params_list)} balances, #{inspect(reason)}" end)
         {:retry, unique_params_list}
     end
   end

apps/indexer/lib/indexer/block_fetcher/catchup.ex

@@ -5,7 +5,6 @@ defmodule Indexer.BlockFetcher.Catchup do
 
   require Logger
 
-  import Indexer, only: [debug: 1]
   import Indexer.BlockFetcher, only: [fetch_and_import_range: 2]
 
   alias Explorer.Chain
@@ -77,7 +76,9 @@ defmodule Indexer.BlockFetcher.Catchup do
           |> Stream.map(&Enum.count/1)
           |> Enum.sum()
 
-        debug(fn -> "#{missing_block_count} missed blocks in #{range_count} ranges between #{first} and #{last}" end)
+        Logger.debug(fn ->
+          "#{missing_block_count} missed blocks in #{range_count} ranges between #{first} and #{last}"
+        end)
 
         case missing_block_count do
           0 ->
@@ -193,7 +194,7 @@ defmodule Indexer.BlockFetcher.Catchup do
   defp cap_seq(seq, next, range) do
     case next do
       :more ->
-        debug(fn ->
+        Logger.debug(fn ->
           first_block_number..last_block_number = range
           "got blocks #{first_block_number} - #{last_block_number}"
         end)

apps/indexer/lib/indexer/block_fetcher/realtime.ex

@@ -8,7 +8,6 @@ defmodule Indexer.BlockFetcher.Realtime do
   require Logger
 
   import EthereumJSONRPC, only: [integer_to_quantity: 1, quantity_to_integer: 1]
-  import Indexer, only: [debug: 1]
   import Indexer.BlockFetcher, only: [fetch_and_import_range: 2]
 
   alias EthereumJSONRPC.Subscription
@@ -65,7 +64,7 @@ defmodule Indexer.BlockFetcher.Realtime do
     # Subscriptions don't support getting all the blocks and transactions data, so we need to go back and get the full block
     case fetch_and_import_range(block_fetcher, number..number) do
       {:ok, {_inserted, _next}} ->
-        debug(fn ->
+        Logger.debug(fn ->
           ["realtime indexer fetched and imported block ", to_string(number)]
         end)
 

apps/indexer/lib/indexer/block_fetcher/receipts.ex

@@ -3,7 +3,7 @@ defmodule Indexer.BlockFetcher.Receipts do
   Fetches transaction receipts after the transactions have been fetched with the blocks in `Indexer.BlockFetcher`.
   """
 
-  import Indexer, only: [debug: 1]
+  require Logger
 
   alias Indexer.BlockFetcher
 
@@ -13,7 +13,7 @@ defmodule Indexer.BlockFetcher.Receipts do
         %BlockFetcher{json_rpc_named_arguments: json_rpc_named_arguments} = state,
         transaction_params
       ) do
-    debug(fn -> "fetching #{length(transaction_params)} transaction receipts" end)
+    Logger.debug(fn -> "fetching #{length(transaction_params)} transaction receipts" end)
     stream_opts = [max_concurrency: state.receipts_concurrency, timeout: :infinity]
 
     transaction_params

apps/indexer/lib/indexer/internal_transaction_fetcher.ex

@@ -88,7 +88,7 @@ defmodule Indexer.InternalTransactionFetcher do
   def run(transactions_params, _retries, json_rpc_named_arguments) do
     unique_transactions_params = unique_transactions_params(transactions_params)
 
-    Indexer.debug(fn -> "fetching internal transactions for #{length(unique_transactions_params)} transactions" end)
+    Logger.debug(fn -> "fetching internal transactions for #{length(unique_transactions_params)} transactions" end)
 
     case EthereumJSONRPC.fetch_internal_transactions(unique_transactions_params, json_rpc_named_arguments) do
       {:ok, internal_transactions_params} ->
@@ -112,7 +112,7 @@ defmodule Indexer.InternalTransactionFetcher do
           |> BalanceFetcher.async_fetch_balances()
         else
           {:error, step, reason, _changes_so_far} ->
-            Indexer.debug(fn ->
+            Logger.debug(fn ->
               [
                 "failed to import internal transactions for ",
                 to_string(length(transactions_params)),
@@ -128,7 +128,7 @@ defmodule Indexer.InternalTransactionFetcher do
         end
 
       {:error, reason} ->
-        Indexer.debug(fn ->
+        Logger.debug(fn ->
           "failed to fetch internal transactions for #{length(transactions_params)} transactions: #{inspect(reason)}"
         end)
 

config/config.exs

@@ -15,6 +15,8 @@ config :logger,
     :console,
     # all applications, but only errors
     {LoggerFileBackend, :error},
+    # only :ecto, but all levels
+    {LoggerFileBackend, :ecto},
     # only :block_scout_web, but all levels
     {LoggerFileBackend, :block_scout_web},
     # only :ethereum_jsonrpc, but all levels
@@ -25,18 +27,22 @@ config :logger,
     {LoggerFileBackend, :indexer}
   ]
 
-# Use same format for all loggers, even though the level should only ever be `:error` for `:error` backend
-format = "$time $metadata[$level] $message\n"
-
-# Configures Elixir's Logger
 config :logger, :console,
-  format: format,
+  # Use same format for all loggers, even though the level should only ever be `:error` for `:error` backend
-  metadata: [:request_id]
+  format: "$time $metadata[$level] $message\n",
+  metadata: [:application, :request_id]
+
+config :logger, :ecto,
+  # Use same format for all loggers, even though the level should only ever be `:error` for `:error` backend
+  format: "$time $metadata[$level] $message\n",
+  metadata: [:application, :request_id],
+  metadata_filter: [application: :ecto]
 
 config :logger, :error,
-  format: format,
+  # Use same format for all loggers, even though the level should only ever be `:error` for `:error` backend
+  format: "$time $metadata[$level] $message\n",
   level: :error,
-  metadata: [:request_id]
+  metadata: [:application, :request_id]
 
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.

config/dev.exs

@@ -1,3 +1,8 @@
 use Mix.Config
 
-config :logger, :console, level: :debug
+# DO NOT make it `:debug` or all Ecto logs will be shown for indexer
+config :logger, :console, level: :info
+
+config :logger, :ecto,
+  level: :debug,
+  path: "logs/dev/ecto.log"

config/prod.exs

@@ -1,4 +1,9 @@
 use Mix.Config
 
 # Do not print debug messages in production
+
 config :logger, :console, level: :info
+
+config :logger, :ecto,
+  level: :info,
+  path: "logs/prod/ecto.log"

config/test.exs

@@ -1,8 +1,13 @@
 use Mix.Config
 
 # Print only warnings and errors during test
+
 config :logger, :console, level: :warn
 
+config :logger, :ecto,
+  level: :warn,
+  path: "logs/test/ecto.log"
+
 config :explorer, Explorer.ExchangeRates,
   source: Explorer.ExchangeRates.Source.NoOpSource,
   store: :none