Merge pull request #841 from poanetwork/836

Add prometheus metric exporting

.credo.exs

@@ -76,7 +76,7 @@
         #
         {Credo.Check.Design.AliasUsage,
          excluded_namespaces: ~w(Socket Task),
-         excluded_lastnames: ~w(Address DateTime Fetcher Full Name Number Repo Time Unit),
+         excluded_lastnames: ~w(Address DateTime Exporter Fetcher Full Instrumenter Name Number Repo Time Unit),
          priority: :low},
 
         # For some checks, you can also set other parameters

README.md

@@ -232,6 +232,35 @@ The app is currently internationalized. It is only localized to U.S. English. To
 `cd apps/block_scout_web; mix gettext.extract --merge; cd -`
 2. To edit the new strings, go to `apps/block_scout_web/priv/gettext/en/LC_MESSAGES/default.po`.
 
+## Metrics
+
+BlockScout is setup to export [Prometheus](https://prometheus.io/) metrics at `/metrics`.
+
+### Prometheus
+
+1. Install prometheus: `brew install prometheus`
+2. Start the web server `iex -S mix phx.server`
+3. Start prometheus: `prometheus --config.file=prometheus.yml`
+
+### Grafana
+
+1. Install grafana: `brew install grafana`
+2. Install Pie Chart panel plugin: `grafana-cli plugins install grafana-piechart-panel`
+3. Start grafana: `brew services start grafana`
+4. Add Prometheus as a Data Source
+   1. `open http://localhost:3000/datasources`
+   2. Click "+ Add data source"
+   3. Put "Prometheus" for "Name"
+   4. Change "Type" to "Prometheus"
+   5. Set "URL" to "http://localhost:9090"
+   6. Set "Scrape Interval" to "10s"
+5. Add the dashboards from https://github.com/deadtrickster/beam-dashboards:
+   For each `*.json` file in the repo.
+   1. `open http://localhost:3000/dashboard/import`
+   2. Copy the contents of the JSON file in the "Or paste JSON" entry
+   3. Click "Load"
+6. View the dashboards.  (You will need to click-around and use BlockScout for the web-related metrics to show up.)
+
 ## Acknowledgements
 
 We would like to thank the [EthPrize foundation](http://ethprize.io/) for their funding support.

apps/block_scout_web/config/config.exs

@@ -14,6 +14,7 @@ config :block_scout_web, BlockScoutWeb.Chain, logo: "/images/poa_logo.svg"
 
 # Configures the endpoint
 config :block_scout_web, BlockScoutWeb.Endpoint,
+  instrumenters: [BlockScoutWeb.Prometheus.Instrumenter],
   url: [host: "localhost"],
   render_errors: [view: BlockScoutWeb.ErrorView, accepts: ~w(html json)],
   pubsub: [name: BlockScoutWeb.PubSub, adapter: Phoenix.PubSub.PG2]

apps/block_scout_web/lib/block_scout_web/application.ex

@@ -5,11 +5,14 @@ defmodule BlockScoutWeb.Application do
 
   use Application
 
-  alias BlockScoutWeb.{Endpoint, EventHandler}
+  alias BlockScoutWeb.{Endpoint, EventHandler, Prometheus}
 
   def start(_type, _args) do
     import Supervisor.Spec
 
+    Prometheus.Instrumenter.setup()
+    Prometheus.Exporter.setup()
+
     # Define workers and child supervisors to be supervised
     children = [
       # Start the endpoint when the application starts

apps/block_scout_web/lib/block_scout_web/endpoint.ex

@@ -51,6 +51,8 @@ defmodule BlockScoutWeb.Endpoint do
     signing_salt: "iC2ksJHS"
   )
 
+  plug(BlockScoutWeb.Prometheus.Exporter)
+
   plug(BlockScoutWeb.Router)
 
   def init(_key, config) do

apps/block_scout_web/lib/block_scout_web/prometheus/exporter.ex

@@ -0,0 +1,9 @@
+defmodule BlockScoutWeb.Prometheus.Exporter do
+  @moduledoc """
+  Exports `Prometheus` metrics at `/metrics`
+  """
+
+  @dialyzer :no_match
+
+  use Prometheus.PlugExporter
+end

apps/block_scout_web/lib/block_scout_web/prometheus/instrumenter.ex

@@ -0,0 +1,16 @@
+defmodule BlockScoutWeb.Prometheus.Instrumenter do
+  @moduledoc """
+  Phoenix request metrics for `Prometheus`.
+  """
+
+  @dialyzer {:no_match,
+             [
+               phoenix_channel_join: 3,
+               phoenix_channel_receive: 3,
+               phoenix_controller_call: 3,
+               phoenix_controller_render: 3,
+               setup: 0
+             ]}
+
+  use Prometheus.PhoenixInstrumenter
+end

apps/block_scout_web/mix.exs

@@ -93,6 +93,14 @@ defmodule BlockScoutWeb.Mixfile do
       # https://github.com/michalmuskala/jason/issues/15
       {:poison, "~> 3.1"},
       {:postgrex, ">= 0.0.0"},
+      # For compatibility with `prometheus_process_collector`, which hasn't been updated yet
+      {:prometheus, "~> 4.0", override: true},
+      # Gather methods for Phoenix requests
+      {:prometheus_phoenix, "~> 1.2"},
+      # Expose metrics from URL Prometheus server can scrape
+      {:prometheus_plugs, "~> 1.1"},
+      # OS process metrics for Prometheus
+      {:prometheus_process_collector, "~> 1.3"},
       {:qrcode, "~> 0.1.0"},
       {:sobelow, ">= 0.7.0", only: [:dev, :test], runtime: false},
       {:timex, "~> 3.1.24"},

apps/explorer/config/config.exs

@@ -18,7 +18,9 @@ config :explorer, Explorer.ExchangeRates, enabled: true
 
 config :explorer, Explorer.Market.History.Cataloger, enabled: true
 
-config :explorer, Explorer.Repo, migration_timestamps: [type: :utc_datetime]
+config :explorer, Explorer.Repo,
+  loggers: [Explorer.Repo.PrometheusLogger, Ecto.LogEntry],
+  migration_timestamps: [type: :utc_datetime]
 
 config :explorer,
   solc_bin_api_url: "https://solc-bin.ethereum.org"

apps/explorer/config/dev.secret.exs.example

@@ -7,7 +7,7 @@ config :explorer, Explorer.Repo,
   hostname: "localhost",
   username: "postgres",
   password: "<REPLACE WITH THE PASSWORD YOU CHOSE>",
-  loggers: [],
+  loggers: [Explorer.Repo.PrometheusLogger],
   pool_size: 20,
   pool_timeout: 60_000,
   timeout: 80_000

apps/explorer/lib/explorer/application.ex

@@ -5,8 +5,12 @@ defmodule Explorer.Application do
 
   use Application
 
+  alias Explorer.Repo.PrometheusLogger
+
   @impl Application
   def start(_type, _args) do
+    PrometheusLogger.setup()
+
     # Children to start in all environments
     base_children = [
       Explorer.Repo,

apps/explorer/lib/explorer/repo/prometheus_logger.ex

@@ -0,0 +1,9 @@
+defmodule Explorer.Repo.PrometheusLogger do
+  @moduledoc """
+  Log `Ecto` query durations as `Prometheus` metrics.
+  """
+
+  @dialyzer {:no_match, [log: 1, setup: 0]}
+
+  use Prometheus.EctoInstrumenter
+end

apps/explorer/mix.exs

@@ -91,6 +91,10 @@ defmodule Explorer.Mixfile do
       {:mock, "~> 0.3.0", only: [:test], runtime: false},
       {:mox, "~> 0.4", only: [:test]},
       {:postgrex, ">= 0.0.0"},
+      # For compatibility with `prometheus_process_collector`, which hasn't been updated yet
+      {:prometheus, "~> 4.0", override: true},
+      # Prometheus metrics for query duration
+      {:prometheus_ecto, "~> 1.3"},
       {:sobelow, ">= 0.7.0", only: [:dev, :test], runtime: false},
       {:timex, "~> 3.1.24"},
       {:timex_ecto, "~> 3.2.1"}

mix.lock

@@ -1,6 +1,7 @@
 %{
   "abi": {:hex, :abi, "0.1.12", "87ae04cb09e2308db7b3c350584dc3934de0e308f6a056ba82be5756b081a1ca", [:mix], [{:exth_crypto, "~> 0.1.4", [hex: :exth_crypto, repo: "hexpm", optional: false]}], "hexpm"},
   "abnf2": {:hex, :abnf2, "0.1.2", "6f8792b8ac3288dba5fc889c2bceae9fe78f74e1a7b36bea9726ffaa9d7bef95", [:mix], []},
+  "accept": {:hex, :accept, "0.3.3", "548ebb6fb2e8b0d170e75bb6123aea6ceecb0189bb1231eeadf52eac08384a97", [:rebar3], [], "hexpm"},
   "bcrypt_elixir": {:hex, :bcrypt_elixir, "1.0.6", "58a865939b3106d5ad4841f660955b958be6db955dda034fbbc1069dbacb97fa", [:make, :mix], [{:elixir_make, "~> 0.4", [hex: :elixir_make, optional: false]}]},
   "benchee": {:hex, :benchee, "0.13.1", "bd93ca05be78bcb6159c7176230efeda2f724f7ffd485515175ca411dff4893e", [:mix], [{:deep_merge, "~> 0.1", [hex: :deep_merge, optional: false]}]},
   "benchee_csv": {:hex, :benchee_csv, "0.8.0", "0ca094677d6e2b2f601b7ee7864b754789ef9d24d079432e5e3d6f4fb83a4d80", [:mix], [{:benchee, "~> 0.12", [hex: :benchee, optional: false]}, {:csv, "~> 2.0", [hex: :csv, optional: false]}]},
@@ -68,6 +69,12 @@
   "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], []},
   "poolboy": {:hex, :poolboy, "1.5.1", "6b46163901cfd0a1b43d692657ed9d7e599853b3b21b95ae5ae0a777cf9b6ca8", [:rebar], []},
   "postgrex": {:hex, :postgrex, "0.13.5", "3d931aba29363e1443da167a4b12f06dcd171103c424de15e5f3fc2ba3e6d9c5", [:mix], [{:connection, "~> 1.0", [hex: :connection, optional: false]}, {:db_connection, "~> 1.1", [hex: :db_connection, optional: false]}, {:decimal, "~> 1.0", [hex: :decimal, optional: false]}]},
+  "prometheus": {:hex, :prometheus, "4.2.0", "06c58bfdfe28d3168b926da614cb9a6d39593deebde648a5480e32dfa3c370e9", [:mix, :rebar3], [], "hexpm"},
+  "prometheus_ecto": {:hex, :prometheus_ecto, "1.3.0", "fe65de16bcfdd6e3c5455194368cd2c7a7034f734a12c12cb9471f4d5e7690c9", [:mix], [{:ecto, "~> 2.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm"},
+  "prometheus_ex": {:hex, :prometheus_ex, "3.0.3", "5d722263bb1f7a9b1d02554de42e61ea672b4e3c07c3f74e23ce35ab5e111cfa", [:mix], [{:prometheus, "~> 4.0", [hex: :prometheus, repo: "hexpm", optional: false]}], "hexpm"},
+  "prometheus_phoenix": {:hex, :prometheus_phoenix, "1.2.1", "964a74dfbc055f781d3a75631e06ce3816a2913976d1df7830283aa3118a797a", [:mix], [{:phoenix, "~> 1.3", [hex: :phoenix, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.3 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm"},
+  "prometheus_plugs": {:hex, :prometheus_plugs, "1.1.5", "25933d48f8af3a5941dd7b621c889749894d8a1082a6ff7c67cc99dec26377c5", [:mix], [{:accept, "~> 0.1", [hex: :accept, repo: "hexpm", optional: false]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}, {:prometheus_process_collector, "~> 1.1", [hex: :prometheus_process_collector, repo: "hexpm", optional: true]}], "hexpm"},
+  "prometheus_process_collector": {:hex, :prometheus_process_collector, "1.3.1", "f6d275bfd292c969908605c1677e2ff9254a8f80d28d17f8e9e5e527e661ce94", [:rebar3], [{:prometheus, "~> 3.4", [hex: :prometheus, repo: "hexpm", optional: false]}], "hexpm"},
   "qrcode": {:hex, :qrcode, "0.1.1", "9ca0d512a3c69a07d28e7660b046f8f3d835ef5462063814a7639a27302f7213", [:mix], []},
   "ranch": {:hex, :ranch, "1.3.2", "e4965a144dc9fbe70e5c077c65e73c57165416a901bd02ea899cfd95aa890986", [:rebar3], []},
   "set_locale": {:git, "https://github.com/minifast/set_locale.git", "da9ae029642bc0fbd9212c2aaf86c0adca70c084", [branch: "master"]},

prometheus.yml

@@ -0,0 +1,6 @@
+scrape_configs:
+  - job_name: block_scout
+    scheme: 'http'
+    scrape_interval: 10s
+    static_configs:
+        - targets: ['localhost:4000']