Merge pull request #841 from poanetwork/836

Add prometheus metric exporting

.credo.exs

@@ -76,7 +76,7 @@
         #
         {Credo.Check.Design.AliasUsage,
          excluded_namespaces: ~w(Socket Task),
-         excluded_lastnames: ~w(Address DateTime Fetcher Full Name Number Repo Time Unit),
+         excluded_lastnames: ~w(Address DateTime Exporter Fetcher Full Instrumenter Name Number Repo Time Unit),
          priority: :low},
 
         # For some checks, you can also set other parameters

README.md

@@ -96,20 +96,20 @@ _Additional runtime options:_
 
 ### Configuring Ethereum Classic and other EVM Chains
 **Note: Most of these modifications will be consolidated into a single file in the future.**
-  
+
   1. Update the import file in `apps/block_scout_web/assets/css/theme/_variables.scss`. There are several preset css files for our supported chains which include Ethereum Classic, Ethereum Mainnet, Ropsten Testnet, Kovan Testnet, POA Core, and POA Sokol. To deploy Ethereum Classic, change the import to `ethereum_classic_variables`.
-  
+
   2. Update the logo file in `apps/block_scout_web/config/config.exs`. To deploy Ethereum Classic, change this file to `classic_ethereum_logo.svg`.
-  
+
   3. Update the `check_origin` configuration in `apps/block_scout_web/config/prod.exs`. This allows realtime events to occur on your endpoint.
-  
+
   4. Update the node configuration. You will need a full tracing node with WebSockets enabled. Make the changes in the following files (dev/prod):
-  
+
    * `apps/explorer/config/dev/parity.exs`
    * `apps/explorer/config/prod/parity.exs`
    * `apps/indexer/config/dev/parity.exs`
    * `apps/indexer/config/prod/parity.exs`
-  
+
   5. Update the dropdown menu in the main navigation `apps/block_scout_web/lib/block_scout_web/templates/layout/_topnav.html.eex`
 
   6. Update the coin in `apps/explorer/config/config.exs`. This will pull relevant information from Coinmarketcap.com.
@@ -232,6 +232,35 @@ The app is currently internationalized. It is only localized to U.S. English. To
 `cd apps/block_scout_web; mix gettext.extract --merge; cd -`
 2. To edit the new strings, go to `apps/block_scout_web/priv/gettext/en/LC_MESSAGES/default.po`.
 
+## Metrics
+
+BlockScout is setup to export [Prometheus](https://prometheus.io/) metrics at `/metrics`.
+
+### Prometheus
+
+1. Install prometheus: `brew install prometheus`
+2. Start the web server `iex -S mix phx.server`
+3. Start prometheus: `prometheus --config.file=prometheus.yml`
+
+### Grafana
+
+1. Install grafana: `brew install grafana`
+2. Install Pie Chart panel plugin: `grafana-cli plugins install grafana-piechart-panel`
+3. Start grafana: `brew services start grafana`
+4. Add Prometheus as a Data Source
+   1. `open http://localhost:3000/datasources`
+   2. Click "+ Add data source"
+   3. Put "Prometheus" for "Name"
+   4. Change "Type" to "Prometheus"
+   5. Set "URL" to "http://localhost:9090"
+   6. Set "Scrape Interval" to "10s"
+5. Add the dashboards from https://github.com/deadtrickster/beam-dashboards:
+   For each `*.json` file in the repo.
+   1. `open http://localhost:3000/dashboard/import`
+   2. Copy the contents of the JSON file in the "Or paste JSON" entry
+   3. Click "Load"
+6. View the dashboards.  (You will need to click-around and use BlockScout for the web-related metrics to show up.)
+
 ## Acknowledgements
 
 We would like to thank the [EthPrize foundation](http://ethprize.io/) for their funding support.

apps/block_scout_web/config/config.exs

@@ -14,6 +14,7 @@ config :block_scout_web, BlockScoutWeb.Chain, logo: "/images/poa_logo.svg"
 
 # Configures the endpoint
 config :block_scout_web, BlockScoutWeb.Endpoint,
+  instrumenters: [BlockScoutWeb.Prometheus.Instrumenter],
   url: [host: "localhost"],
   render_errors: [view: BlockScoutWeb.ErrorView, accepts: ~w(html json)],
   pubsub: [name: BlockScoutWeb.PubSub, adapter: Phoenix.PubSub.PG2]

apps/block_scout_web/lib/block_scout_web/application.ex

@@ -5,11 +5,14 @@ defmodule BlockScoutWeb.Application do
 
   use Application
 
-  alias BlockScoutWeb.{Endpoint, EventHandler}
+  alias BlockScoutWeb.{Endpoint, EventHandler, Prometheus}
 
   def start(_type, _args) do
     import Supervisor.Spec
 
+    Prometheus.Instrumenter.setup()
+    Prometheus.Exporter.setup()
+
     # Define workers and child supervisors to be supervised
     children = [
       # Start the endpoint when the application starts

apps/block_scout_web/lib/block_scout_web/endpoint.ex

@@ -51,6 +51,8 @@ defmodule BlockScoutWeb.Endpoint do
     signing_salt: "iC2ksJHS"
   )
 
+  plug(BlockScoutWeb.Prometheus.Exporter)
+
   plug(BlockScoutWeb.Router)
 
   def init(_key, config) do

apps/block_scout_web/lib/block_scout_web/prometheus/exporter.ex

@@ -0,0 +1,9 @@
+defmodule BlockScoutWeb.Prometheus.Exporter do
+  @moduledoc """
+  Exports `Prometheus` metrics at `/metrics`
+  """
+
+  @dialyzer :no_match
+
+  use Prometheus.PlugExporter
+end

apps/block_scout_web/lib/block_scout_web/prometheus/instrumenter.ex

@@ -0,0 +1,16 @@
+defmodule BlockScoutWeb.Prometheus.Instrumenter do
+  @moduledoc """
+  Phoenix request metrics for `Prometheus`.
+  """
+
+  @dialyzer {:no_match,
+             [
+               phoenix_channel_join: 3,
+               phoenix_channel_receive: 3,
+               phoenix_controller_call: 3,
+               phoenix_controller_render: 3,
+               setup: 0
+             ]}
+
+  use Prometheus.PhoenixInstrumenter
+end

apps/block_scout_web/mix.exs

@@ -93,6 +93,14 @@ defmodule BlockScoutWeb.Mixfile do
       # https://github.com/michalmuskala/jason/issues/15
       {:poison, "~> 3.1"},
       {:postgrex, ">= 0.0.0"},
+      # For compatibility with `prometheus_process_collector`, which hasn't been updated yet
+      {:prometheus, "~> 4.0", override: true},
+      # Gather methods for Phoenix requests
+      {:prometheus_phoenix, "~> 1.2"},
+      # Expose metrics from URL Prometheus server can scrape
+      {:prometheus_plugs, "~> 1.1"},
+      # OS process metrics for Prometheus
+      {:prometheus_process_collector, "~> 1.3"},
       {:qrcode, "~> 0.1.0"},
       {:sobelow, ">= 0.7.0", only: [:dev, :test], runtime: false},
       {:timex, "~> 3.1.24"},

apps/explorer/config/config.exs

@@ -18,7 +18,9 @@ config :explorer, Explorer.ExchangeRates, enabled: true
 
 config :explorer, Explorer.Market.History.Cataloger, enabled: true
 
-config :explorer, Explorer.Repo, migration_timestamps: [type: :utc_datetime]
+config :explorer, Explorer.Repo,
+  loggers: [Explorer.Repo.PrometheusLogger, Ecto.LogEntry],
+  migration_timestamps: [type: :utc_datetime]
 
 config :explorer,
   solc_bin_api_url: "https://solc-bin.ethereum.org"

apps/explorer/config/dev.secret.exs.example

@@ -7,7 +7,7 @@ config :explorer, Explorer.Repo,
   hostname: "localhost",
   username: "postgres",
   password: "<REPLACE WITH THE PASSWORD YOU CHOSE>",
-  loggers: [],
+  loggers: [Explorer.Repo.PrometheusLogger],
   pool_size: 20,
   pool_timeout: 60_000,
   timeout: 80_000

apps/explorer/lib/explorer/application.ex

@@ -5,8 +5,12 @@ defmodule Explorer.Application do
 
   use Application
 
+  alias Explorer.Repo.PrometheusLogger
+
   @impl Application
   def start(_type, _args) do
+    PrometheusLogger.setup()
+
     # Children to start in all environments
     base_children = [
       Explorer.Repo,

apps/explorer/lib/explorer/repo/prometheus_logger.ex

@@ -0,0 +1,9 @@
+defmodule Explorer.Repo.PrometheusLogger do
+  @moduledoc """
+  Log `Ecto` query durations as `Prometheus` metrics.
+  """
+
+  @dialyzer {:no_match, [log: 1, setup: 0]}
+
+  use Prometheus.EctoInstrumenter
+end

apps/explorer/mix.exs

@@ -91,6 +91,10 @@ defmodule Explorer.Mixfile do
       {:mock, "~> 0.3.0", only: [:test], runtime: false},
       {:mox, "~> 0.4", only: [:test]},
       {:postgrex, ">= 0.0.0"},
+      # For compatibility with `prometheus_process_collector`, which hasn't been updated yet
+      {:prometheus, "~> 4.0", override: true},
+      # Prometheus metrics for query duration
+      {:prometheus_ecto, "~> 1.3"},
       {:sobelow, ">= 0.7.0", only: [:dev, :test], runtime: false},
       {:timex, "~> 3.1.24"},
       {:timex_ecto, "~> 3.2.1"}

mix.lock

@@ -1,6 +1,7 @@
 %{
   "abi": {:hex, :abi, "0.1.12", "87ae04cb09e2308db7b3c350584dc3934de0e308f6a056ba82be5756b081a1ca", [:mix], [{:exth_crypto, "~> 0.1.4", [hex: :exth_crypto, repo: "hexpm", optional: false]}], "hexpm"},
   "abnf2": {:hex, :abnf2, "0.1.2", "6f8792b8ac3288dba5fc889c2bceae9fe78f74e1a7b36bea9726ffaa9d7bef95", [:mix], []},
+  "accept": {:hex, :accept, "0.3.3", "548ebb6fb2e8b0d170e75bb6123aea6ceecb0189bb1231eeadf52eac08384a97", [:rebar3], [], "hexpm"},
   "bcrypt_elixir": {:hex, :bcrypt_elixir, "1.0.6", "58a865939b3106d5ad4841f660955b958be6db955dda034fbbc1069dbacb97fa", [:make, :mix], [{:elixir_make, "~> 0.4", [hex: :elixir_make, optional: false]}]},
   "benchee": {:hex, :benchee, "0.13.1", "bd93ca05be78bcb6159c7176230efeda2f724f7ffd485515175ca411dff4893e", [:mix], [{:deep_merge, "~> 0.1", [hex: :deep_merge, optional: false]}]},
   "benchee_csv": {:hex, :benchee_csv, "0.8.0", "0ca094677d6e2b2f601b7ee7864b754789ef9d24d079432e5e3d6f4fb83a4d80", [:mix], [{:benchee, "~> 0.12", [hex: :benchee, optional: false]}, {:csv, "~> 2.0", [hex: :csv, optional: false]}]},
@@ -68,6 +69,12 @@
   "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], []},
   "poolboy": {:hex, :poolboy, "1.5.1", "6b46163901cfd0a1b43d692657ed9d7e599853b3b21b95ae5ae0a777cf9b6ca8", [:rebar], []},
   "postgrex": {:hex, :postgrex, "0.13.5", "3d931aba29363e1443da167a4b12f06dcd171103c424de15e5f3fc2ba3e6d9c5", [:mix], [{:connection, "~> 1.0", [hex: :connection, optional: false]}, {:db_connection, "~> 1.1", [hex: :db_connection, optional: false]}, {:decimal, "~> 1.0", [hex: :decimal, optional: false]}]},
+  "prometheus": {:hex, :prometheus, "4.2.0", "06c58bfdfe28d3168b926da614cb9a6d39593deebde648a5480e32dfa3c370e9", [:mix, :rebar3], [], "hexpm"},
+  "prometheus_ecto": {:hex, :prometheus_ecto, "1.3.0", "fe65de16bcfdd6e3c5455194368cd2c7a7034f734a12c12cb9471f4d5e7690c9", [:mix], [{:ecto, "~> 2.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm"},
+  "prometheus_ex": {:hex, :prometheus_ex, "3.0.3", "5d722263bb1f7a9b1d02554de42e61ea672b4e3c07c3f74e23ce35ab5e111cfa", [:mix], [{:prometheus, "~> 4.0", [hex: :prometheus, repo: "hexpm", optional: false]}], "hexpm"},
+  "prometheus_phoenix": {:hex, :prometheus_phoenix, "1.2.1", "964a74dfbc055f781d3a75631e06ce3816a2913976d1df7830283aa3118a797a", [:mix], [{:phoenix, "~> 1.3", [hex: :phoenix, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.3 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm"},
+  "prometheus_plugs": {:hex, :prometheus_plugs, "1.1.5", "25933d48f8af3a5941dd7b621c889749894d8a1082a6ff7c67cc99dec26377c5", [:mix], [{:accept, "~> 0.1", [hex: :accept, repo: "hexpm", optional: false]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}, {:prometheus_process_collector, "~> 1.1", [hex: :prometheus_process_collector, repo: "hexpm", optional: true]}], "hexpm"},
+  "prometheus_process_collector": {:hex, :prometheus_process_collector, "1.3.1", "f6d275bfd292c969908605c1677e2ff9254a8f80d28d17f8e9e5e527e661ce94", [:rebar3], [{:prometheus, "~> 3.4", [hex: :prometheus, repo: "hexpm", optional: false]}], "hexpm"},
   "qrcode": {:hex, :qrcode, "0.1.1", "9ca0d512a3c69a07d28e7660b046f8f3d835ef5462063814a7639a27302f7213", [:mix], []},
   "ranch": {:hex, :ranch, "1.3.2", "e4965a144dc9fbe70e5c077c65e73c57165416a901bd02ea899cfd95aa890986", [:rebar3], []},
   "set_locale": {:git, "https://github.com/minifast/set_locale.git", "da9ae029642bc0fbd9212c2aaf86c0adca70c084", [branch: "master"]},

prometheus.yml

@@ -0,0 +1,6 @@
+scrape_configs:
+  - job_name: block_scout
+    scheme: 'http'
+    scrape_interval: 10s
+    static_configs:
+        - targets: ['localhost:4000']