Improved logging

Have separate loggers for errors and per-application.

* `:console` will continue to have all applications and levels.  Level is
  - `:debug` for `dev`
  - `:warn` for `test`
  - `:info` for `prod`
* `:error` is only for errors across all environments.  It uses
  `LoggerFileBackend` and writes to `logs/<Mix.env>/error.log`.
* `:block_scout_web` is only when `:application` metadata is
  `:block_scout_web`.  Level is
  - `:debug` for `dev`
  - `:warn` for `test`
  - `:info` for `prod`
  It uses `LoggerFileBackend` and writes to
`logs/<Mix.env>/block_scout_web.log`.
* `:ethereum_jsonrpc` is only when `:application` metadata is
  `:ethereum_jsonrpc`.  Level is
  - `:debug` for `dev`
  - `:warn` for `test`
  - `:info` for `prod`
  It uses `LoggerFileBackend` and writes to
`logs/<Mix.env>/ethereum_jsonrpc.log`.
* `:explorer` is only when `:application` metadata is `:exploer`.  Level is
  - `:debug` for `dev`
  - `:warn` for `test`
  - `:info` for `prod`
  It uses `LoggerFileBackend` and writes to `logs/<Mix.env>/explorer.log`.
* `:indexer` is only when `:application` metadata is `:indexer`.  Level is
  - `:debug` for `dev`
  - `:warn` for `test`
  - `:info` for `prod`
  It uses `LoggerFileBackend` and writes to `logs/<Mix.env>/indexer.log`.

.gitignore

@@ -6,6 +6,7 @@
 /deps
 /doc
 /*.ez
+/logs
 
 # Generated on crash by the VM
 erl_crash.dump

apps/block_scout_web/config/config.exs

@@ -32,6 +32,12 @@ config :ex_cldr,
   locales: ["en"],
   gettext: BlockScoutWeb.Gettext
 
+config :logger, :block_scout_web,
+  # keep synced with `config/config.exs`
+  format: "$time $metadata[$level] $message\n",
+  metadata: [:request_id],
+  metadata_filter: [application: :block_scout_web]
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"

apps/block_scout_web/config/dev.exs

@@ -48,6 +48,10 @@ config :block_scout_web, BlockScoutWeb.Endpoint,
     ]
   ]
 
+config :logger, :block_scout_web,
+  level: :debug,
+  path: "logs/dev/block_scout_web.log"
+
 # Set a higher stacktrace during development. Avoid configuring such
 # in production as building large stacktraces may be expensive.
 config :phoenix, :stacktrace_depth, 20

apps/block_scout_web/config/prod.exs

@@ -23,3 +23,7 @@ config :block_scout_web, BlockScoutWeb.Endpoint,
     scheme: "http",
     port: System.get_env("PORT")
   ]
+
+config :logger, :block_scout_web,
+  level: :info,
+  path: "logs/prod/block_scout_web.log"

apps/block_scout_web/config/test.exs

@@ -9,6 +9,10 @@ config :block_scout_web, BlockScoutWeb.Endpoint,
   secret_key_base: "27Swe6KtEtmN37WyEYRjKWyxYULNtrxlkCEKur4qoV+Lwtk8lafsR16ifz1XBBYj",
   server: true
 
+config :logger, :block_scout_web,
+  level: :warn,
+  path: "logs/test/block_scout_web.log"
+
 # Configure wallaby
 config :wallaby, screenshot_on_failure: true
 

apps/block_scout_web/mix.exs

@@ -79,6 +79,8 @@ defmodule BlockScoutWeb.Mixfile do
       {:gettext, "~> 0.14.1"},
       {:httpoison, "~> 1.0", override: true},
       {:junit_formatter, ">= 0.0.0", only: [:test], runtime: false},
+      # Log errors and application output to separate files
+      {:logger_file_backend, "~> 0.0.10"},
       {:math, "~> 0.3.0"},
       {:mock, "~> 0.3.0", only: [:test], runtime: false},
       {:phoenix, "~> 1.3.0"},

apps/ethereum_jsonrpc/config/config.exs

@@ -0,0 +1,11 @@
+use Mix.Config
+
+config :logger, :ethereum_jsonrpc,
+  # keep synced with `config/config.exs`
+  format: "$time $metadata[$level] $message\n",
+  metadata: [:request_id],
+  metadata_filter: [application: :ethereum_jsonrpc]
+
+# Import environment specific config. This must remain at the bottom
+# of this file so it overrides the configuration defined above.
+import_config "#{Mix.env()}.exs"

apps/ethereum_jsonrpc/config/dev.exs

@@ -0,0 +1,5 @@
+use Mix.Config
+
+config :logger, :ethereum_jsonrpc,
+  level: :debug,
+  path: "logs/dev/ethereum_jsonrpc.log"

apps/ethereum_jsonrpc/config/prod.exs

@@ -0,0 +1,5 @@
+use Mix.Config
+
+config :logger, :ethereum_jsonrpc,
+  level: :info,
+  path: "logs/prod/ethereum_jsonrpc.log"

apps/ethereum_jsonrpc/config/test.exs

@@ -0,0 +1,5 @@
+use Mix.Config
+
+config :logger, :ethereum_jsonrpc,
+  level: :warn,
+  path: "logs/test/ethereum_jsonrpc.log"

apps/ethereum_jsonrpc/config/test/mox.exs

@@ -1,5 +0,0 @@
-# Tests with everything using `Mox`
-
-use Mix.Config
-
-config :ethereum_jsonrpc, EthereumJSONRPC.Case, json_rpc_named_arguments: [transport: EthereumJSONRPC.Mox]

apps/ethereum_jsonrpc/mix.exs

@@ -69,6 +69,8 @@ defmodule EthereumJsonrpc.MixProject do
       {:httpoison, "~> 1.0", override: true},
       # Decode/Encode JSON for JSONRPC
       {:jason, "~> 1.0"},
+      # Log errors and application output to separate files
+      {:logger_file_backend, "~> 0.0.10"},
       # Mocking `EthereumJSONRPC.Transport` and `EthereumJSONRPC.HTTP` so we avoid hitting real chains for local testing
       {:mox, "~> 0.4", only: [:test]},
       # Convert unix timestamps in JSONRPC to DateTimes

apps/explorer/config/config.exs

@@ -23,6 +23,12 @@ config :explorer, Explorer.Repo, migration_timestamps: [type: :utc_datetime]
 config :explorer,
   solc_bin_api_url: "https://solc-bin.ethereum.org"
 
+config :logger, :explorer,
+  # keep synced with `config/config.exs`
+  format: "$time $metadata[$level] $message\n",
+  metadata: [:request_id],
+  metadata_filter: [application: :explorer]
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"

apps/explorer/config/dev.exs

@@ -10,6 +10,10 @@ config :explorer, Explorer.Repo,
   pool_timeout: 60_000,
   timeout: 80_000
 
+config :logger, :explorer,
+  level: :debug,
+  path: "logs/dev/explorer.log"
+
 import_config "dev.secret.exs"
 
 variant =

apps/explorer/config/prod.exs

@@ -9,6 +9,10 @@ config :explorer, Explorer.Repo,
   prepare: :unnamed,
   timeout: 60_000
 
+config :logger, :explorer,
+  level: :info,
+  path: "logs/prod/explorer.log"
+
 variant =
   if is_nil(System.get_env("ETHEREUM_JSONRPC_VARIANT")) do
     "parity"

apps/explorer/config/test.exs

@@ -17,6 +17,10 @@ config :explorer, Explorer.ExchangeRates, enabled: false
 
 config :explorer, Explorer.Market.History.Cataloger, enabled: false
 
+config :logger, :explorer,
+  level: :warn,
+  path: "logs/test/explorer.log"
+
 if File.exists?(file = "test.secret.exs") do
   import_config file
 end

apps/explorer/mix.exs

@@ -81,6 +81,8 @@ defmodule Explorer.Mixfile do
       {:httpoison, "~> 1.0", override: true},
       {:jason, "~> 1.0"},
       {:junit_formatter, ">= 0.0.0", only: [:test], runtime: false},
+      # Log errors and application output to separate files
+      {:logger_file_backend, "~> 0.0.10"},
       {:math, "~> 0.3.0"},
       {:mock, "~> 0.3.0", only: [:test], runtime: false},
       {:mox, "~> 0.4", only: [:test]},

apps/indexer/config/config.exs

@@ -6,6 +6,12 @@ config :indexer,
   debug_logs: !!System.get_env("DEBUG_INDEXER"),
   ecto_repos: [Explorer.Repo]
 
+config :logger, :indexer,
+  # keep synced with `config/config.exs`
+  format: "$time $metadata[$level] $message\n",
+  metadata: [:request_id],
+  metadata_filter: [application: :indexer]
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"

apps/indexer/config/dev.exs

@@ -1,5 +1,9 @@
 use Mix.Config
 
+config :logger, :indexer,
+  level: :debug,
+  path: "logs/dev/indexer.log"
+
 variant =
   if is_nil(System.get_env("ETHEREUM_JSONRPC_VARIANT")) do
     "parity"

apps/indexer/config/prod.exs

@@ -1,5 +1,9 @@
 use Mix.Config
 
+config :logger, :indexer,
+  level: :info,
+  path: "logs/prod/indexer.log"
+
 variant =
   if is_nil(System.get_env("ETHEREUM_JSONRPC_VARIANT")) do
     "parity"

apps/indexer/config/test.exs

@@ -1 +1,5 @@
 use Mix.Config
+
+config :logger, :indexer,
+  level: :warn,
+  path: "logs/test/indexer.log"

apps/indexer/mix.exs

@@ -40,6 +40,8 @@ defmodule Indexer.MixProject do
       {:ethereum_jsonrpc, in_umbrella: true},
       # Importing to database
       {:explorer, in_umbrella: true},
+      # Log errors and application output to separate files
+      {:logger_file_backend, "~> 0.0.10"},
       # Mocking `EthereumJSONRPC.Transport`, so we avoid hitting real chains for local testing
       {:mox, "~> 0.4", only: [:test]}
     ]

config/config.exs

@@ -9,9 +9,33 @@ use Mix.Config
 # back to each application for organization purposes.
 import_config "../apps/*/config/config.exs"
 
+config :logger,
+  backends: [
+    # all applications and all levels
+    :console,
+    # all applications, but only errors
+    {LoggerFileBackend, :error},
+    # only :block_scout_web, but all levels
+    {LoggerFileBackend, :block_scout_web},
+    # only :ethereum_jsonrpc, but all levels
+    {LoggerFileBackend, :ethereum_jsonrpc},
+    # only :explorer, but all levels
+    {LoggerFileBackend, :explorer},
+    # only :indexer, but all levels
+    {LoggerFileBackend, :indexer}
+  ]
+
+# Use same format for all loggers, even though the level should only ever be `:error` for `:error` backend
+format = "$time $metadata[$level] $message\n"
+
 # Configures Elixir's Logger
 config :logger, :console,
-  format: "$time $metadata[$level] $message\n",
+  format: format,
+  metadata: [:request_id]
+
+config :logger, :error,
+  format: format,
+  level: :error,
   metadata: [:request_id]
 
 # Import environment specific config. This must remain at the bottom

config/dev.exs

@@ -1,5 +1,3 @@
 use Mix.Config
 
-# Do not include metadata nor timestamps in development logs
+config :logger, :console, level: :debug
-config :logger, :console, format: "[$level] $message\n"
-config :logger, level: :debug

config/prod.exs

@@ -1,4 +1,4 @@
 use Mix.Config
 
 # Do not print debug messages in production
-config :logger, level: :info
+config :logger, :console, level: :info

config/test.exs

@@ -1,7 +1,7 @@
 use Mix.Config
 
 # Print only warnings and errors during test
-config :logger, level: :warn
+config :logger, :console, level: :warn
 
 config :explorer, Explorer.ExchangeRates,
   source: Explorer.ExchangeRates.Source.NoOpSource,

mix.lock

@@ -30,7 +30,7 @@
   "ex_doc": {:hex, :ex_doc, "0.18.3", "f4b0e4a2ec6f333dccf761838a4b253d75e11f714b85ae271c9ae361367897b7", [:mix], [{:earmark, "~> 1.1", [hex: :earmark, optional: false]}]},
   "ex_machina": {:hex, :ex_machina, "2.1.0", "4874dc9c78e7cf2d429f24dc3c4005674d4e4da6a08be961ffccc08fb528e28b", [:mix], [{:ecto, "~> 2.1", [hex: :ecto, optional: true]}]},
   "exactor": {:hex, :exactor, "2.2.4", "5efb4ddeb2c48d9a1d7c9b465a6fffdd82300eb9618ece5d34c3334d5d7245b1", [:mix], []},
-  "excoveralls": {:hex, :excoveralls, "0.8.1", "0bbf67f22c7dbf7503981d21a5eef5db8bbc3cb86e70d3798e8c802c74fa5e27", [:mix], [{:exjsx, ">= 3.0.0", [hex: :exjsx, optional: false]}, {:hackney, ">= 0.12.0", [hex: :hackney, optional: false]}]},
+  "excoveralls": {:hex, :excoveralls, "0.8.1", "0bbf67f22c7dbf7503981d21a5eef5db8bbc3cb86e70d3798e8c802c74fa5e27", [:mix], [{:exjsx, ">= 3.0.0", [repo: "hexpm", hex: :exjsx, optional: false]}, {:hackney, ">= 0.12.0", [repo: "hexpm", hex: :hackney, optional: false]}], "hexpm"},
   "exjsx": {:hex, :exjsx, "4.0.0", "60548841e0212df401e38e63c0078ec57b33e7ea49b032c796ccad8cde794b5c", [:mix], [{:jsx, "~> 2.8.0", [hex: :jsx, optional: false]}]},
   "exth_crypto": {:hex, :exth_crypto, "0.1.4", "11f9084dfd70d4f9e96f2710a472f4e6b23044b97530c719550c2b0450ffeb61", [:mix], [{:binary, "~> 0.0.4", [hex: :binary, optional: false]}, {:keccakf1600, "~> 2.0.0", [hex: :keccakf1600_orig, optional: false]}, {:libsecp256k1, "~> 0.1.3", [hex: :libsecp256k1, optional: false]}]},
   "exvcr": {:hex, :exvcr, "0.10.0", "5150808404d9f48dbda636f70f7f8fefd93e2433cd39f695f810e73b3a9d1736", [:mix], [{:exactor, "~> 2.2", [hex: :exactor, optional: false]}, {:exjsx, "~> 4.0", [hex: :exjsx, optional: false]}, {:httpoison, "~> 0.13", [hex: :httpoison, optional: true]}, {:httpotion, "~> 3.0", [hex: :httpotion, optional: true]}, {:ibrowse, "~> 4.4", [hex: :ibrowse, optional: true]}, {:meck, "~> 0.8.8", [hex: :meck, optional: false]}]},
@@ -39,15 +39,16 @@
   "flow": {:hex, :flow, "0.12.0", "32c5a5f3ff6693e004b6c17a8c64dce2f8cdaf9564912d79427176013a586ab6", [:mix], [{:gen_stage, "~> 0.12.0", [hex: :gen_stage, optional: false]}]},
   "gen_stage": {:hex, :gen_stage, "0.12.2", "e0e347cbb1ceb5f4e68a526aec4d64b54ad721f0a8b30aa9d28e0ad749419cbb", [:mix], []},
   "gettext": {:hex, :gettext, "0.14.1", "a666b6782fdb6ccb4736170ccb0928c0f773aa0e47c694d4b7d0338f724ff189", [:mix], []},
-  "hackney": {:hex, :hackney, "1.12.1", "8bf2d0e11e722e533903fe126e14d6e7e94d9b7983ced595b75f532e04b7fdc7", [:rebar3], [{:certifi, "2.3.1", [hex: :certifi, optional: false]}, {:idna, "5.1.1", [hex: :idna, optional: false]}, {:metrics, "1.0.1", [hex: :metrics, optional: false]}, {:mimerl, "1.0.2", [hex: :mimerl, optional: false]}, {:ssl_verify_fun, "1.1.1", [hex: :ssl_verify_fun, optional: false]}]},
+  "hackney": {:hex, :hackney, "1.12.1", "8bf2d0e11e722e533903fe126e14d6e7e94d9b7983ced595b75f532e04b7fdc7", [:rebar3], [{:certifi, "2.3.1", [repo: "hexpm", hex: :certifi, optional: false]}, {:idna, "5.1.1", [repo: "hexpm", hex: :idna, optional: false]}, {:metrics, "1.0.1", [repo: "hexpm", hex: :metrics, optional: false]}, {:mimerl, "1.0.2", [repo: "hexpm", hex: :mimerl, optional: false]}, {:ssl_verify_fun, "1.1.1", [repo: "hexpm", hex: :ssl_verify_fun, optional: false]}], "hexpm"},
   "html_entities": {:hex, :html_entities, "0.4.0", "f2fee876858cf6aaa9db608820a3209e45a087c5177332799592142b50e89a6b", [:mix], []},
   "httpoison": {:hex, :httpoison, "1.1.0", "497949fb62924432f64a45269d20e6f61ecf35084ffa270917afcdb7cd4d8061", [:mix], [{:hackney, "~> 1.8", [hex: :hackney, optional: false]}]},
-  "idna": {:hex, :idna, "5.1.1", "cbc3b2fa1645113267cc59c760bafa64b2ea0334635ef06dbac8801e42f7279c", [:rebar3], [{:unicode_util_compat, "0.3.1", [hex: :unicode_util_compat, optional: false]}]},
+  "idna": {:hex, :idna, "5.1.1", "cbc3b2fa1645113267cc59c760bafa64b2ea0334635ef06dbac8801e42f7279c", [:rebar3], [{:unicode_util_compat, "0.3.1", [repo: "hexpm", hex: :unicode_util_compat, optional: false]}], "hexpm"},
   "jason": {:hex, :jason, "1.0.0", "0f7cfa9bdb23fed721ec05419bcee2b2c21a77e926bce0deda029b5adc716fe2", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, optional: true]}]},
   "jsx": {:hex, :jsx, "2.8.3", "a05252d381885240744d955fbe3cf810504eb2567164824e19303ea59eef62cf", [:mix, :rebar3], []},
   "junit_formatter": {:hex, :junit_formatter, "2.1.0", "ff03d2bbe9a67041f2488d8e72180ddcf4dff9e9c8a39b79eac9828fcb9e9bbf", [:mix], []},
   "keccakf1600": {:hex, :keccakf1600_orig, "2.0.0", "0a7217ddb3ee8220d449bbf7575ec39d4e967099f220a91e3dfca4dbaef91963", [:rebar3], []},
   "libsecp256k1": {:hex, :libsecp256k1, "0.1.4", "42b7f76d8e32f85f578ccda0abfdb1afa0c5c231d1fd8aeab9cda352731a2d83", [:rebar3], []},
+  "logger_file_backend": {:hex, :logger_file_backend, "0.0.10", "876f9f84ae110781207c54321ffbb62bebe02946fe3c13f0d7c5f5d8ad4fa910", [:mix], [], "hexpm"},
   "math": {:hex, :math, "0.3.0", "e14e7291115201cb155a3567e66d196bf5088a6f55b030d598107d7ae934a11c", [:mix], []},
   "meck": {:hex, :meck, "0.8.9", "64c5c0bd8bcca3a180b44196265c8ed7594e16bcc845d0698ec6b4e577f48188", [:rebar3], []},
   "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], []},