From fdd1090c454070c7435581288726067cd19e3f9b Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Mon, 27 Aug 2018 14:03:04 -0500 Subject: [PATCH] Re-enable Ecto loggers Allow Ecto loggers to log to the application-specific log **file**, `log//ecto.log` but raise `:console`'s level to `:info`, so that Ecto output does not flood `iex` and make it unusable. --- README.md | 2 +- apps/block_scout_web/config/config.exs | 2 +- apps/ethereum_jsonrpc/config/config.exs | 2 +- apps/explorer/config/config.exs | 2 +- apps/explorer/config/dev.exs | 1 - apps/indexer/config/config.exs | 3 +- apps/indexer/lib/indexer.ex | 30 ------------------- apps/indexer/lib/indexer/balance_fetcher.ex | 6 ++-- .../lib/indexer/block_fetcher/catchup.ex | 7 +++-- .../lib/indexer/block_fetcher/realtime.ex | 3 +- .../lib/indexer/block_fetcher/receipts.ex | 4 +-- .../indexer/internal_transaction_fetcher.ex | 6 ++-- config/config.exs | 22 +++++++++----- config/dev.exs | 7 ++++- config/prod.exs | 5 ++++ config/test.exs | 5 ++++ 16 files changed, 49 insertions(+), 58 deletions(-) diff --git a/README.md b/README.md index b821e64044..e71c29ecc5 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ _Additional runtime options:_ `iex -S mix phx.server` * Run Phoenix Server with real time indexer -`DEBUG_INDEXER=1 iex -S mix phx.server` +`iex -S mix phx.server` ### BlockScout Visual Interface diff --git a/apps/block_scout_web/config/config.exs b/apps/block_scout_web/config/config.exs index 5b35edd8f2..edcac0d8c9 100644 --- a/apps/block_scout_web/config/config.exs +++ b/apps/block_scout_web/config/config.exs @@ -35,7 +35,7 @@ config :ex_cldr, config :logger, :block_scout_web, # keep synced with `config/config.exs` format: "$time $metadata[$level] $message\n", - metadata: [:request_id], + metadata: [:application, :request_id], metadata_filter: [application: :block_scout_web] # Import environment specific config. This must remain at the bottom diff --git a/apps/ethereum_jsonrpc/config/config.exs b/apps/ethereum_jsonrpc/config/config.exs index 84a166757b..f89de8c121 100644 --- a/apps/ethereum_jsonrpc/config/config.exs +++ b/apps/ethereum_jsonrpc/config/config.exs @@ -3,7 +3,7 @@ use Mix.Config config :logger, :ethereum_jsonrpc, # keep synced with `config/config.exs` format: "$time $metadata[$level] $message\n", - metadata: [:request_id], + metadata: [:application, :request_id], metadata_filter: [application: :ethereum_jsonrpc] # Import environment specific config. This must remain at the bottom diff --git a/apps/explorer/config/config.exs b/apps/explorer/config/config.exs index 41ccb8b68f..5b2927c576 100644 --- a/apps/explorer/config/config.exs +++ b/apps/explorer/config/config.exs @@ -26,7 +26,7 @@ config :explorer, config :logger, :explorer, # keep synced with `config/config.exs` format: "$time $metadata[$level] $message\n", - metadata: [:request_id], + metadata: [:application, :request_id], metadata_filter: [application: :explorer] # Import environment specific config. This must remain at the bottom diff --git a/apps/explorer/config/dev.exs b/apps/explorer/config/dev.exs index 85d04021ba..2cf332eab3 100644 --- a/apps/explorer/config/dev.exs +++ b/apps/explorer/config/dev.exs @@ -5,7 +5,6 @@ config :explorer, Explorer.Repo, adapter: Ecto.Adapters.Postgres, database: "explorer_dev", hostname: "localhost", - loggers: [], pool_size: 20, pool_timeout: 60_000, timeout: 80_000 diff --git a/apps/indexer/config/config.exs b/apps/indexer/config/config.exs index 95ae69bde3..0b4041ac14 100644 --- a/apps/indexer/config/config.exs +++ b/apps/indexer/config/config.exs @@ -3,13 +3,12 @@ use Mix.Config config :indexer, - debug_logs: !!System.get_env("DEBUG_INDEXER"), ecto_repos: [Explorer.Repo] config :logger, :indexer, # keep synced with `config/config.exs` format: "$time $metadata[$level] $message\n", - metadata: [:request_id], + metadata: [:application, :request_id], metadata_filter: [application: :indexer] # Import environment specific config. This must remain at the bottom diff --git a/apps/indexer/lib/indexer.ex b/apps/indexer/lib/indexer.ex index 44979554a7..47c0d7b9a8 100644 --- a/apps/indexer/lib/indexer.ex +++ b/apps/indexer/lib/indexer.ex @@ -2,7 +2,6 @@ defmodule Indexer do @moduledoc """ Indexes an Ethereum-based chain using JSONRPC. """ - require Logger alias Explorer.Chain @@ -53,33 +52,4 @@ defmodule Indexer do num -> num + 1 end end - - @doc """ - Logs debug message if `:debug_logs` have been enabled. - """ - def debug(message_func) when is_function(message_func, 0) do - if debug_logs_enabled?() do - Logger.debug(message_func) - else - :noop - end - end - - @doc """ - Enables debug logs for indexing system. - """ - def enable_debug_logs do - Application.put_env(:indexer, :debug_logs, true) - end - - @doc """ - Disables debug logs for indexing system. - """ - def disable_debug_logs do - Application.put_env(:indexer, :debug_logs, false) - end - - defp debug_logs_enabled? do - Application.fetch_env!(:indexer, :debug_logs) - end end diff --git a/apps/indexer/lib/indexer/balance_fetcher.ex b/apps/indexer/lib/indexer/balance_fetcher.ex index 50f882a6e3..fd98e02fb9 100644 --- a/apps/indexer/lib/indexer/balance_fetcher.ex +++ b/apps/indexer/lib/indexer/balance_fetcher.ex @@ -4,6 +4,8 @@ defmodule Indexer.BalanceFetcher do `fetched_balance_block_number` to value at max `t:Explorer.Chain.Balance.t/0` `block_number` for the given `t:Explorer.Chain.Address.t/` `hash`. """ + require Logger + import EthereumJSONRPC, only: [integer_to_quantity: 1] alias Explorer.Chain @@ -68,7 +70,7 @@ defmodule Indexer.BalanceFetcher do # `{address, block}`, so take unique params only unique_params_list = Enum.uniq(params_list) - Indexer.debug(fn -> "fetching #{length(unique_params_list)} balances" end) + Logger.debug(fn -> "fetching #{length(unique_params_list)} balances" end) case EthereumJSONRPC.fetch_balances(unique_params_list, json_rpc_named_arguments) do {:ok, balances_params} -> @@ -87,7 +89,7 @@ defmodule Indexer.BalanceFetcher do :ok {:error, reason} -> - Indexer.debug(fn -> "failed to fetch #{length(unique_params_list)} balances, #{inspect(reason)}" end) + Logger.debug(fn -> "failed to fetch #{length(unique_params_list)} balances, #{inspect(reason)}" end) {:retry, unique_params_list} end end diff --git a/apps/indexer/lib/indexer/block_fetcher/catchup.ex b/apps/indexer/lib/indexer/block_fetcher/catchup.ex index 9ab836cce5..e47cf5eb6b 100644 --- a/apps/indexer/lib/indexer/block_fetcher/catchup.ex +++ b/apps/indexer/lib/indexer/block_fetcher/catchup.ex @@ -5,7 +5,6 @@ defmodule Indexer.BlockFetcher.Catchup do require Logger - import Indexer, only: [debug: 1] import Indexer.BlockFetcher, only: [fetch_and_import_range: 2] alias Explorer.Chain @@ -77,7 +76,9 @@ defmodule Indexer.BlockFetcher.Catchup do |> Stream.map(&Enum.count/1) |> Enum.sum() - debug(fn -> "#{missing_block_count} missed blocks in #{range_count} ranges between #{first} and #{last}" end) + Logger.debug(fn -> + "#{missing_block_count} missed blocks in #{range_count} ranges between #{first} and #{last}" + end) case missing_block_count do 0 -> @@ -193,7 +194,7 @@ defmodule Indexer.BlockFetcher.Catchup do defp cap_seq(seq, next, range) do case next do :more -> - debug(fn -> + Logger.debug(fn -> first_block_number..last_block_number = range "got blocks #{first_block_number} - #{last_block_number}" end) diff --git a/apps/indexer/lib/indexer/block_fetcher/realtime.ex b/apps/indexer/lib/indexer/block_fetcher/realtime.ex index 60f6e68624..37e0d7e874 100644 --- a/apps/indexer/lib/indexer/block_fetcher/realtime.ex +++ b/apps/indexer/lib/indexer/block_fetcher/realtime.ex @@ -8,7 +8,6 @@ defmodule Indexer.BlockFetcher.Realtime do require Logger import EthereumJSONRPC, only: [integer_to_quantity: 1, quantity_to_integer: 1] - import Indexer, only: [debug: 1] import Indexer.BlockFetcher, only: [fetch_and_import_range: 2] alias EthereumJSONRPC.Subscription @@ -65,7 +64,7 @@ defmodule Indexer.BlockFetcher.Realtime do # Subscriptions don't support getting all the blocks and transactions data, so we need to go back and get the full block case fetch_and_import_range(block_fetcher, number..number) do {:ok, {_inserted, _next}} -> - debug(fn -> + Logger.debug(fn -> ["realtime indexer fetched and imported block ", to_string(number)] end) diff --git a/apps/indexer/lib/indexer/block_fetcher/receipts.ex b/apps/indexer/lib/indexer/block_fetcher/receipts.ex index a50ba001b1..51171e7c3a 100644 --- a/apps/indexer/lib/indexer/block_fetcher/receipts.ex +++ b/apps/indexer/lib/indexer/block_fetcher/receipts.ex @@ -3,7 +3,7 @@ defmodule Indexer.BlockFetcher.Receipts do Fetches transaction receipts after the transactions have been fetched with the blocks in `Indexer.BlockFetcher`. """ - import Indexer, only: [debug: 1] + require Logger alias Indexer.BlockFetcher @@ -13,7 +13,7 @@ defmodule Indexer.BlockFetcher.Receipts do %BlockFetcher{json_rpc_named_arguments: json_rpc_named_arguments} = state, transaction_params ) do - debug(fn -> "fetching #{length(transaction_params)} transaction receipts" end) + Logger.debug(fn -> "fetching #{length(transaction_params)} transaction receipts" end) stream_opts = [max_concurrency: state.receipts_concurrency, timeout: :infinity] transaction_params diff --git a/apps/indexer/lib/indexer/internal_transaction_fetcher.ex b/apps/indexer/lib/indexer/internal_transaction_fetcher.ex index 45907e7ccb..5d30904cd5 100644 --- a/apps/indexer/lib/indexer/internal_transaction_fetcher.ex +++ b/apps/indexer/lib/indexer/internal_transaction_fetcher.ex @@ -88,7 +88,7 @@ defmodule Indexer.InternalTransactionFetcher do def run(transactions_params, _retries, json_rpc_named_arguments) do unique_transactions_params = unique_transactions_params(transactions_params) - Indexer.debug(fn -> "fetching internal transactions for #{length(unique_transactions_params)} transactions" end) + Logger.debug(fn -> "fetching internal transactions for #{length(unique_transactions_params)} transactions" end) case EthereumJSONRPC.fetch_internal_transactions(unique_transactions_params, json_rpc_named_arguments) do {:ok, internal_transactions_params} -> @@ -112,7 +112,7 @@ defmodule Indexer.InternalTransactionFetcher do |> BalanceFetcher.async_fetch_balances() else {:error, step, reason, _changes_so_far} -> - Indexer.debug(fn -> + Logger.debug(fn -> [ "failed to import internal transactions for ", to_string(length(transactions_params)), @@ -128,7 +128,7 @@ defmodule Indexer.InternalTransactionFetcher do end {:error, reason} -> - Indexer.debug(fn -> + Logger.debug(fn -> "failed to fetch internal transactions for #{length(transactions_params)} transactions: #{inspect(reason)}" end) diff --git a/config/config.exs b/config/config.exs index ea164a8ac6..50400ef963 100644 --- a/config/config.exs +++ b/config/config.exs @@ -15,6 +15,8 @@ config :logger, :console, # all applications, but only errors {LoggerFileBackend, :error}, + # only :ecto, but all levels + {LoggerFileBackend, :ecto}, # only :block_scout_web, but all levels {LoggerFileBackend, :block_scout_web}, # only :ethereum_jsonrpc, but all levels @@ -25,18 +27,22 @@ config :logger, {LoggerFileBackend, :indexer} ] -# Use same format for all loggers, even though the level should only ever be `:error` for `:error` backend -format = "$time $metadata[$level] $message\n" - -# Configures Elixir's Logger config :logger, :console, - format: format, - metadata: [:request_id] + # Use same format for all loggers, even though the level should only ever be `:error` for `:error` backend + format: "$time $metadata[$level] $message\n", + metadata: [:application, :request_id] + +config :logger, :ecto, + # Use same format for all loggers, even though the level should only ever be `:error` for `:error` backend + format: "$time $metadata[$level] $message\n", + metadata: [:application, :request_id], + metadata_filter: [application: :ecto] config :logger, :error, - format: format, + # Use same format for all loggers, even though the level should only ever be `:error` for `:error` backend + format: "$time $metadata[$level] $message\n", level: :error, - metadata: [:request_id] + metadata: [:application, :request_id] # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. diff --git a/config/dev.exs b/config/dev.exs index f69ee58e56..5d5249dc46 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -1,3 +1,8 @@ use Mix.Config -config :logger, :console, level: :debug +# DO NOT make it `:debug` or all Ecto logs will be shown for indexer +config :logger, :console, level: :info + +config :logger, :ecto, + level: :debug, + path: "logs/dev/ecto.log" diff --git a/config/prod.exs b/config/prod.exs index e6ce408bd9..afb69eb4e0 100644 --- a/config/prod.exs +++ b/config/prod.exs @@ -1,4 +1,9 @@ use Mix.Config # Do not print debug messages in production + config :logger, :console, level: :info + +config :logger, :ecto, + level: :info, + path: "logs/prod/ecto.log" diff --git a/config/test.exs b/config/test.exs index 5a4fec994c..a5510715c5 100644 --- a/config/test.exs +++ b/config/test.exs @@ -1,8 +1,13 @@ use Mix.Config # Print only warnings and errors during test + config :logger, :console, level: :warn +config :logger, :ecto, + level: :warn, + path: "logs/test/ecto.log" + config :explorer, Explorer.ExchangeRates, source: Explorer.ExchangeRates.Source.NoOpSource, store: :none