diff --git a/.credo.exs b/.credo.exs
index ceb9421811..7dc6c59a95 100644
--- a/.credo.exs
+++ b/.credo.exs
@@ -75,8 +75,9 @@
# Priority values are: `low, normal, high, higher`
#
{Credo.Check.Design.AliasUsage,
- excluded_namespaces: ~w(Block Blocks Import Socket Task),
- excluded_lastnames: ~w(Address DateTime Exporter Fetcher Full Instrumenter Monitor Name Number Repo Time Unit),
+ excluded_namespaces: ~w(Block Blocks Import Socket SpandexDatadog Task),
+ excluded_lastnames:
+ ~w(Address DateTime Exporter Fetcher Full Instrumenter Monitor Name Number Repo Spec Time Unit),
priority: :low},
# For some checks, you can also set other parameters
diff --git a/README.md b/README.md
index f201753977..2d10d4ce0f 100644
--- a/README.md
+++ b/README.md
@@ -7,7 +7,7 @@
BlockScout
Blockchain Explorer for inspecting and analyzing EVM Chains.
-
+
[![CircleCI](https://circleci.com/gh/poanetwork/blockscout.svg?style=svg&circle-token=f8823a3d0090407c11f87028c73015a331dbf604)](https://circleci.com/gh/poanetwork/blockscout) [![Coverage Status](https://coveralls.io/repos/github/poanetwork/blockscout/badge.svg?branch=master)](https://coveralls.io/github/poanetwork/blockscout?branch=master) [![Join the chat at https://gitter.im/poanetwork/blockscout](https://badges.gitter.im/poanetwork/blockscout.svg)](https://gitter.im/poanetwork/blockscout?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
@@ -283,6 +283,33 @@ BlockScout is setup to export [Prometheus](https://prometheus.io/) metrics at `/
3. Click "Load"
6. View the dashboards. (You will need to click-around and use BlockScout for the web-related metrics to show up.)
+## Tracing
+
+Blockscout supports tracing via
+[Spandex](http://git@github.com:spandex-project/spandex.git). Each application
+has its own tracer, that is configured internally to that application. In order
+to enable it, visit each application's `config/.ex` and update its tracer
+configuration to change `disabled?: true` to `disabled?: false`. Do this for
+each application you'd like included in your trace data.
+
+Currently, only [Datadog](https://www.datadoghq.com/) is supported as a
+tracing backend, but more will be added soon.
+
+### DataDog
+
+If you would like to use DataDog, after enabling `Spandex`, set
+`"DATADOG_HOST"` and `"DATADOG_PORT"` environment variables to the
+host/port that your Datadog agent is running on. For more information on
+Datadog and the Datadog agent, see their
+[documentation](https://docs.datadoghq.com/).
+
+### Other
+
+If you want to use a different backend, remove the
+`SpandexDatadog.ApiServer` `Supervisor.child_spec` from
+`Explorer.Application` and follow any instructions provided in `Spandex`
+for setting up that backend.
+
## Memory Usage
The work queues for building the index of all blocks, balances (coin and token), and internal transactions can grow quite large. By default, the soft-limit is 1 GiB, which can be changed in `apps/indexer/config/config.exs`:
diff --git a/apps/block_scout_web/config/config.exs b/apps/block_scout_web/config/config.exs
index 3f3eecb4e1..05b7a3f1a5 100644
--- a/apps/block_scout_web/config/config.exs
+++ b/apps/block_scout_web/config/config.exs
@@ -18,7 +18,7 @@ config :block_scout_web, BlockScoutWeb.Chain,
# Configures the endpoint
config :block_scout_web, BlockScoutWeb.Endpoint,
- instrumenters: [BlockScoutWeb.Prometheus.Instrumenter],
+ instrumenters: [BlockScoutWeb.Prometheus.Instrumenter, SpandexPhoenix.Instrumenter],
url: [
host: "localhost",
path: System.get_env("NETWORK_PATH") || "/"
@@ -26,6 +26,11 @@ config :block_scout_web, BlockScoutWeb.Endpoint,
render_errors: [view: BlockScoutWeb.ErrorView, accepts: ~w(html json)],
pubsub: [name: BlockScoutWeb.PubSub, adapter: Phoenix.PubSub.PG2]
+config :block_scout_web, BlockScoutWeb.Tracer,
+ service: :block_scout_web,
+ adapter: SpandexDatadog.Adapter,
+ trace_key: :blockscout
+
# Configures gettext
config :block_scout_web, BlockScoutWeb.Gettext, locales: ~w(en), default_locale: "en"
@@ -46,6 +51,8 @@ config :logger, :block_scout_web,
metadata: [:application, :request_id],
metadata_filter: [application: :block_scout_web]
+config :spandex_phoenix, tracer: BlockScoutWeb.Tracer
+
config :wobserver,
# return only the local node
discovery: :none,
diff --git a/apps/block_scout_web/config/dev.exs b/apps/block_scout_web/config/dev.exs
index d821ac555f..2b50183667 100644
--- a/apps/block_scout_web/config/dev.exs
+++ b/apps/block_scout_web/config/dev.exs
@@ -48,6 +48,8 @@ config :block_scout_web, BlockScoutWeb.Endpoint,
]
]
+config :block_scout_web, BlockScoutWeb.Tracer, env: "dev", disabled?: true
+
config :logger, :block_scout_web,
level: :debug,
path: Path.absname("logs/dev/block_scout_web.log")
diff --git a/apps/block_scout_web/config/prod.exs b/apps/block_scout_web/config/prod.exs
index a6de66fea4..1fb27bdbed 100644
--- a/apps/block_scout_web/config/prod.exs
+++ b/apps/block_scout_web/config/prod.exs
@@ -24,6 +24,8 @@ config :block_scout_web, BlockScoutWeb.Endpoint,
port: System.get_env("PORT")
]
+config :block_scout_web, BlockScoutWeb.Tracer, env: "production", disabled?: true
+
config :logger, :block_scout_web,
level: :info,
path: Path.absname("logs/prod/block_scout_web.log"),
diff --git a/apps/block_scout_web/config/test.exs b/apps/block_scout_web/config/test.exs
index 74c119ab45..531ea3f1e6 100644
--- a/apps/block_scout_web/config/test.exs
+++ b/apps/block_scout_web/config/test.exs
@@ -9,6 +9,8 @@ config :block_scout_web, BlockScoutWeb.Endpoint,
secret_key_base: "27Swe6KtEtmN37WyEYRjKWyxYULNtrxlkCEKur4qoV+Lwtk8lafsR16ifz1XBBYj",
server: true
+config :block_scout_web, BlockScoutWeb.Tracer, disabled?: false
+
config :logger, :block_scout_web,
level: :warn,
path: Path.absname("logs/test/block_scout_web.log")
diff --git a/apps/block_scout_web/lib/block_scout_web/endpoint.ex b/apps/block_scout_web/lib/block_scout_web/endpoint.ex
index 673f3ea7c6..caed03e3ff 100644
--- a/apps/block_scout_web/lib/block_scout_web/endpoint.ex
+++ b/apps/block_scout_web/lib/block_scout_web/endpoint.ex
@@ -68,6 +68,8 @@ defmodule BlockScoutWeb.Endpoint do
signing_salt: "iC2ksJHS"
)
+ use SpandexPhoenix
+
plug(BlockScoutWeb.Prometheus.Exporter)
plug(BlockScoutWeb.Router)
diff --git a/apps/block_scout_web/lib/block_scout_web/tracer.ex b/apps/block_scout_web/lib/block_scout_web/tracer.ex
new file mode 100644
index 0000000000..6bca34d564
--- /dev/null
+++ b/apps/block_scout_web/lib/block_scout_web/tracer.ex
@@ -0,0 +1,5 @@
+defmodule BlockScoutWeb.Tracer do
+ @moduledoc false
+
+ use Spandex.Tracer, otp_app: :block_scout_web
+end
diff --git a/apps/block_scout_web/mix.exs b/apps/block_scout_web/mix.exs
index d7a1be661e..22a7624add 100644
--- a/apps/block_scout_web/mix.exs
+++ b/apps/block_scout_web/mix.exs
@@ -112,6 +112,12 @@ defmodule BlockScoutWeb.Mixfile do
{:prometheus_process_collector, "~> 1.3"},
{:qrcode, "~> 0.1.0"},
{:sobelow, ">= 0.7.0", only: [:dev, :test], runtime: false},
+ # Tracing
+ {:spandex, github: "spandex-project/spandex", branch: "allow-setting-trace-key", override: true},
+ # `:spandex` integration with Datadog
+ {:spandex_datadog, "~> 0.3.1"},
+ # `:spandex` tracing of `:phoenix`
+ {:spandex_phoenix, "~> 0.3.0"},
{:timex, "~> 3.4"},
{:wallaby, "~> 0.20", only: [:test], runtime: false},
{:wobserver, "~> 0.1.8"}
diff --git a/apps/ethereum_jsonrpc/config/config.exs b/apps/ethereum_jsonrpc/config/config.exs
index 782c1ae1ff..1725d5abe9 100644
--- a/apps/ethereum_jsonrpc/config/config.exs
+++ b/apps/ethereum_jsonrpc/config/config.exs
@@ -1,11 +1,5 @@
use Mix.Config
-config :logger, :ethereum_jsonrpc,
- # keep synced with `config/config.exs`
- format: "$time $metadata[$level] $message\n",
- metadata: [:application, :request_id],
- metadata_filter: [application: :ethereum_jsonrpc]
-
config :ethereum_jsonrpc, EthereumJSONRPC.RequestCoordinator,
rolling_window_opts: [
window_count: 12,
@@ -15,6 +9,17 @@ config :ethereum_jsonrpc, EthereumJSONRPC.RequestCoordinator,
wait_per_timeout: :timer.seconds(20),
max_jitter: :timer.seconds(2)
+config :ethereum_jsonrpc, EthereumJSONRPC.Tracer,
+ service: :ethereum_jsonrpc,
+ adapter: SpandexDatadog.Adapter,
+ trace_key: :blockscout
+
+config :logger, :ethereum_jsonrpc,
+ # keep synced with `config/config.exs`
+ format: "$time $metadata[$level] $message\n",
+ metadata: [:application, :request_id],
+ metadata_filter: [application: :ethereum_jsonrpc]
+
# Import environment specific config. This must remain at the bottom
# of this file so it overrides the configuration defined above.
import_config "#{Mix.env()}.exs"
diff --git a/apps/ethereum_jsonrpc/config/dev.exs b/apps/ethereum_jsonrpc/config/dev.exs
index 1e5d0ebbe4..1f81a56cb5 100644
--- a/apps/ethereum_jsonrpc/config/dev.exs
+++ b/apps/ethereum_jsonrpc/config/dev.exs
@@ -1,5 +1,7 @@
use Mix.Config
+config :ethereum_jsonrpc, EthereumJSONRPC.Tracer, env: "dev", disabled?: true
+
config :logger, :ethereum_jsonrpc,
level: :debug,
path: Path.absname("logs/dev/ethereum_jsonrpc.log")
diff --git a/apps/ethereum_jsonrpc/config/prod.exs b/apps/ethereum_jsonrpc/config/prod.exs
index 0f1324c682..4b4d80babb 100644
--- a/apps/ethereum_jsonrpc/config/prod.exs
+++ b/apps/ethereum_jsonrpc/config/prod.exs
@@ -1,5 +1,7 @@
use Mix.Config
+config :ethereum_jsonrpc, EthereumJSONRPC.Tracer, env: "production", disabled?: true
+
config :logger, :ethereum_jsonrpc,
level: :info,
path: Path.absname("logs/prod/ethereum_jsonrpc.log"),
diff --git a/apps/ethereum_jsonrpc/config/test.exs b/apps/ethereum_jsonrpc/config/test.exs
index cf1666a448..e76eab248f 100644
--- a/apps/ethereum_jsonrpc/config/test.exs
+++ b/apps/ethereum_jsonrpc/config/test.exs
@@ -1,9 +1,5 @@
use Mix.Config
-config :logger, :ethereum_jsonrpc,
- level: :warn,
- path: Path.absname("logs/test/ethereum_jsonrpc.log")
-
config :ethereum_jsonrpc, EthereumJSONRPC.RequestCoordinator,
rolling_window_opts: [
window_count: 3,
@@ -12,3 +8,9 @@ config :ethereum_jsonrpc, EthereumJSONRPC.RequestCoordinator,
],
wait_per_timeout: 2,
max_jitter: 1
+
+config :ethereum_jsonrpc, EthereumJSONRPC.Tracer, disabled?: false
+
+config :logger, :ethereum_jsonrpc,
+ level: :warn,
+ path: Path.absname("logs/test/ethereum_jsonrpc.log")
diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/request_coordinator.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/request_coordinator.ex
index 8e8a767d76..c72953555b 100644
--- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/request_coordinator.ex
+++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/request_coordinator.ex
@@ -42,7 +42,9 @@ defmodule EthereumJSONRPC.RequestCoordinator do
With this configuration, timeouts are tracked for 6 windows of 10 seconds for a total of 1 minute.
"""
- alias EthereumJSONRPC.{RollingWindow, Transport}
+ require EthereumJSONRPC.Tracer
+
+ alias EthereumJSONRPC.{RollingWindow, Tracer, Transport}
@error_key :throttleable_error_count
@@ -63,9 +65,11 @@ defmodule EthereumJSONRPC.RequestCoordinator do
if sleep_time <= throttle_timeout do
:timer.sleep(sleep_time)
- request
- |> transport.json_rpc(transport_options)
- |> handle_transport_response()
+ trace_request(request, fn ->
+ request
+ |> transport.json_rpc(transport_options)
+ |> handle_transport_response()
+ end)
else
:timer.sleep(throttle_timeout)
@@ -73,6 +77,18 @@ defmodule EthereumJSONRPC.RequestCoordinator do
end
end
+ defp trace_request([request | _], fun) do
+ trace_request(request, fun)
+ end
+
+ defp trace_request(%{method: method}, fun) do
+ Tracer.span "RequestCoordinator.perform/4", resource: method, service: :ethereum_jsonrpc do
+ fun.()
+ end
+ end
+
+ defp trace_request(_, fun), do: fun.()
+
defp handle_transport_response({:error, {:bad_gateway, _}} = error) do
RollingWindow.inc(table(), @error_key)
error
diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/tracer.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/tracer.ex
new file mode 100644
index 0000000000..159ed37bbb
--- /dev/null
+++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/tracer.ex
@@ -0,0 +1,5 @@
+defmodule EthereumJSONRPC.Tracer do
+ @moduledoc false
+
+ use Spandex.Tracer, otp_app: :ethereum_jsonrpc
+end
diff --git a/apps/ethereum_jsonrpc/mix.exs b/apps/ethereum_jsonrpc/mix.exs
index 02c8ccfe53..d91458c4ab 100644
--- a/apps/ethereum_jsonrpc/mix.exs
+++ b/apps/ethereum_jsonrpc/mix.exs
@@ -76,6 +76,10 @@ defmodule EthereumJsonrpc.MixProject do
{:logger_file_backend, "~> 0.0.10"},
# Mocking `EthereumJSONRPC.Transport` and `EthereumJSONRPC.HTTP` so we avoid hitting real chains for local testing
{:mox, "~> 0.4", only: [:test]},
+ # Tracing
+ {:spandex, github: "spandex-project/spandex", branch: "allow-setting-trace-key", override: true},
+ # `:spandex` integration with Datadog
+ {:spandex_datadog, "~> 0.3.1"},
# Convert unix timestamps in JSONRPC to DateTimes
{:timex, "~> 3.4"},
# Encode/decode function names and arguments
diff --git a/apps/explorer/config/config.exs b/apps/explorer/config/config.exs
index a4af332015..b5892e3525 100644
--- a/apps/explorer/config/config.exs
+++ b/apps/explorer/config/config.exs
@@ -25,6 +25,11 @@ config :explorer, Explorer.Repo,
loggers: [Explorer.Repo.PrometheusLogger, Ecto.LogEntry],
migration_timestamps: [type: :utc_datetime]
+config :explorer, Explorer.Tracer,
+ service: :explorer,
+ adapter: SpandexDatadog.Adapter,
+ trace_key: :blockscout
+
config :explorer, Explorer.Counters.TokenTransferCounter, enabled: true
config :explorer, Explorer.Counters.TokenHoldersCounter, enabled: true, enable_consolidation: true
@@ -46,6 +51,11 @@ config :logger, :explorer,
metadata: [:application, :request_id],
metadata_filter: [application: :explorer]
+config :spandex_ecto, SpandexEcto.EctoLogger,
+ service: :ecto,
+ tracer: Explorer.Tracer,
+ otp_app: :explorer
+
# Import environment specific config. This must remain at the bottom
# of this file so it overrides the configuration defined above.
import_config "#{Mix.env()}.exs"
diff --git a/apps/explorer/config/dev.exs b/apps/explorer/config/dev.exs
index 8cc1f28a5f..2e61bb9124 100644
--- a/apps/explorer/config/dev.exs
+++ b/apps/explorer/config/dev.exs
@@ -9,6 +9,8 @@ config :explorer, Explorer.Repo,
pool_timeout: 60_000,
timeout: 80_000
+config :explorer, Explorer.Tracer, env: "dev", disabled?: true
+
config :logger, :explorer,
level: :debug,
path: Path.absname("logs/dev/explorer.log")
diff --git a/apps/explorer/config/prod.exs b/apps/explorer/config/prod.exs
index bc8e95d9db..4d5fe15c46 100644
--- a/apps/explorer/config/prod.exs
+++ b/apps/explorer/config/prod.exs
@@ -9,6 +9,8 @@ config :explorer, Explorer.Repo,
prepare: :unnamed,
timeout: 60_000
+config :explorer, Explorer.Tracer, env: "production", disabled?: true
+
config :logger, :explorer,
level: :info,
path: Path.absname("logs/prod/explorer.log"),
diff --git a/apps/explorer/config/test.exs b/apps/explorer/config/test.exs
index 764f2f6ed1..f0e4330a01 100644
--- a/apps/explorer/config/test.exs
+++ b/apps/explorer/config/test.exs
@@ -3,6 +3,8 @@ use Mix.Config
# Lower hashing rounds for faster tests
config :bcrypt_elixir, log_rounds: 4
+config :explorer, Explorer.Counters.TokenHoldersCounter, enabled: true, enable_consolidation: false
+
# Configure your database
config :explorer, Explorer.Repo,
adapter: Ecto.Adapters.Postgres,
@@ -17,7 +19,7 @@ config :explorer, Explorer.ExchangeRates, enabled: false
config :explorer, Explorer.Market.History.Cataloger, enabled: false
-config :explorer, Explorer.Counters.TokenHoldersCounter, enabled: true, enable_consolidation: false
+config :explorer, Explorer.Tracer, disabled?: false
config :logger, :explorer,
level: :warn,
diff --git a/apps/explorer/lib/explorer/application.ex b/apps/explorer/lib/explorer/application.ex
index dac4ce0c8a..665c9b08a0 100644
--- a/apps/explorer/lib/explorer/application.ex
+++ b/apps/explorer/lib/explorer/application.ex
@@ -15,6 +15,7 @@ defmodule Explorer.Application do
# Children to start in all environments
base_children = [
Explorer.Repo,
+ Supervisor.Spec.worker(SpandexDatadog.ApiServer, [datadog_opts()]),
Supervisor.child_spec({Task.Supervisor, name: Explorer.MarketTaskSupervisor}, id: Explorer.MarketTaskSupervisor),
Supervisor.child_spec({Task.Supervisor, name: Explorer.TaskSupervisor}, id: Explorer.TaskSupervisor),
{Registry, keys: :duplicate, name: Registry.ChainEvents, id: Registry.ChainEvents},
@@ -52,4 +53,14 @@ defmodule Explorer.Application do
[]
end
end
+
+ defp datadog_opts do
+ [
+ host: System.get_env("DATADOG_HOST") || "localhost",
+ port: System.get_env("DATADOG_PORT") || 8126,
+ batch_size: System.get_env("SPANDEX_BATCH_SIZE") || 100,
+ sync_threshold: System.get_env("SPANDEX_SYNC_THRESHOLD") || 100,
+ http: HTTPoison
+ ]
+ end
end
diff --git a/apps/explorer/lib/explorer/tracer.ex b/apps/explorer/lib/explorer/tracer.ex
new file mode 100644
index 0000000000..20297d80ab
--- /dev/null
+++ b/apps/explorer/lib/explorer/tracer.ex
@@ -0,0 +1,5 @@
+defmodule Explorer.Tracer do
+ @moduledoc false
+
+ use Spandex.Tracer, otp_app: :explorer
+end
diff --git a/apps/explorer/mix.exs b/apps/explorer/mix.exs
index 90da255dd0..e7b0a6263f 100644
--- a/apps/explorer/mix.exs
+++ b/apps/explorer/mix.exs
@@ -96,6 +96,12 @@ defmodule Explorer.Mixfile do
# bypass optional dependency
{:plug_cowboy, "~> 1.0", only: :test},
{:sobelow, ">= 0.7.0", only: [:dev, :test], runtime: false},
+ # Tracing
+ {:spandex, github: "spandex-project/spandex", branch: "allow-setting-trace-key", override: true},
+ # `:spandex` integration with Datadog
+ {:spandex_datadog, "~> 0.3.1"},
+ # `:spandex` tracing of `:ecto`
+ {:spandex_ecto, "~> 0.4.0"},
{:timex, "~> 3.4"},
# `Timex.Duration` for `Explorer.Chain.average_block_time/0`
{:timex_ecto, "~> 3.3"}
diff --git a/apps/indexer/config/config.exs b/apps/indexer/config/config.exs
index e19bcd3610..0225befbf3 100644
--- a/apps/indexer/config/config.exs
+++ b/apps/indexer/config/config.exs
@@ -11,6 +11,11 @@ config :indexer,
# bytes
memory_limit: 1 <<< 30
+config :indexer, Indexer.Tracer,
+ service: :indexer,
+ adapter: SpandexDatadog.Adapter,
+ trace_key: :blockscout
+
config :logger, :indexer,
# keep synced with `config/config.exs`
format: "$time $metadata[$level] $message\n",
diff --git a/apps/indexer/config/dev.exs b/apps/indexer/config/dev.exs
index 337e2f9b97..ffd1d6b0ee 100644
--- a/apps/indexer/config/dev.exs
+++ b/apps/indexer/config/dev.exs
@@ -1,5 +1,7 @@
use Mix.Config
+config :indexer, Indexer.Tracer, env: "dev", disabled?: true
+
config :logger, :indexer,
level: :debug,
path: Path.absname("logs/dev/indexer.log")
diff --git a/apps/indexer/config/prod.exs b/apps/indexer/config/prod.exs
index f342efa867..49800bcc15 100644
--- a/apps/indexer/config/prod.exs
+++ b/apps/indexer/config/prod.exs
@@ -1,5 +1,7 @@
use Mix.Config
+config :indexer, Indexer.Tracer, env: "production", disabled?: true
+
config :logger, :indexer,
level: :info,
path: Path.absname("logs/prod/indexer.log"),
diff --git a/apps/indexer/config/test.exs b/apps/indexer/config/test.exs
index e1ba98f859..e9c5505405 100644
--- a/apps/indexer/config/test.exs
+++ b/apps/indexer/config/test.exs
@@ -1,5 +1,7 @@
use Mix.Config
+config :indexer, Indexer.Tracer, disabled?: false
+
config :logger, :indexer,
level: :warn,
path: Path.absname("logs/test/indexer.log")
diff --git a/apps/indexer/lib/indexer/block/catchup/fetcher.ex b/apps/indexer/lib/indexer/block/catchup/fetcher.ex
index 79611270aa..56b96e761f 100644
--- a/apps/indexer/lib/indexer/block/catchup/fetcher.ex
+++ b/apps/indexer/lib/indexer/block/catchup/fetcher.ex
@@ -3,13 +3,15 @@ defmodule Indexer.Block.Catchup.Fetcher do
Fetches and indexes block ranges from the block before the latest block to genesis (0) that are missing.
"""
+ use Spandex.Decorators
+
require Logger
import Indexer.Block.Fetcher,
only: [async_import_coin_balances: 2, async_import_tokens: 1, async_import_uncles: 1, fetch_and_import_range: 2]
alias Explorer.Chain
- alias Indexer.{Block, InternalTransaction, Sequence, TokenBalance}
+ alias Indexer.{Block, InternalTransaction, Sequence, TokenBalance, Tracer}
alias Indexer.Memory.Shrinkable
@behaviour Block.Fetcher
@@ -160,6 +162,11 @@ defmodule Indexer.Block.Catchup.Fetcher do
end
# Run at state.blocks_concurrency max_concurrency when called by `stream_import/1`
+ @decorate trace(
+ name: "fetch",
+ resource: "Indexer.Block.Catchup.Fetcher.fetch_and_import_range_from_sequence/3",
+ tracer: Tracer
+ )
defp fetch_and_import_range_from_sequence(
%__MODULE__{block_fetcher: %Block.Fetcher{} = block_fetcher},
_.._ = range,
diff --git a/apps/indexer/lib/indexer/block/fetcher.ex b/apps/indexer/lib/indexer/block/fetcher.ex
index 5e7585b59c..1d48135f6e 100644
--- a/apps/indexer/lib/indexer/block/fetcher.ex
+++ b/apps/indexer/lib/indexer/block/fetcher.ex
@@ -3,11 +3,13 @@ defmodule Indexer.Block.Fetcher do
Fetches and indexes block ranges.
"""
+ use Spandex.Decorators
+
require Logger
alias EthereumJSONRPC.{Blocks, FetchedBeneficiaries}
alias Explorer.Chain.{Address, Block, Import}
- alias Indexer.{AddressExtraction, CoinBalance, MintTransfer, Token, TokenTransfers}
+ alias Indexer.{AddressExtraction, CoinBalance, MintTransfer, Token, TokenTransfers, Tracer}
alias Indexer.Address.{CoinBalances, TokenBalances}
alias Indexer.Block.Fetcher.Receipts
alias Indexer.Block.Transform
@@ -76,6 +78,7 @@ defmodule Indexer.Block.Fetcher do
struct!(__MODULE__, named_arguments)
end
+ @decorate span(tracer: Tracer)
@spec fetch_and_import_range(t, Range.t()) ::
{:ok, %{inserted: %{}, errors: [EthereumJSONRPC.Transport.error()]}}
| {:error,
diff --git a/apps/indexer/lib/indexer/block/realtime/fetcher.ex b/apps/indexer/lib/indexer/block/realtime/fetcher.ex
index e1667679af..f9a37ef47b 100644
--- a/apps/indexer/lib/indexer/block/realtime/fetcher.ex
+++ b/apps/indexer/lib/indexer/block/realtime/fetcher.ex
@@ -4,7 +4,9 @@ defmodule Indexer.Block.Realtime.Fetcher do
"""
use GenServer
+ use Spandex.Decorators
+ require Indexer.Tracer
require Logger
import EthereumJSONRPC, only: [integer_to_quantity: 1, quantity_to_integer: 1]
@@ -13,7 +15,7 @@ defmodule Indexer.Block.Realtime.Fetcher do
alias Ecto.Changeset
alias EthereumJSONRPC.{FetchedBalances, Subscription}
alias Explorer.Chain
- alias Indexer.{AddressExtraction, Block, TokenBalances}
+ alias Indexer.{AddressExtraction, Block, TokenBalances, Tracer}
alias Indexer.Block.Realtime.TaskSupervisor
@behaviour Block.Fetcher
@@ -125,7 +127,13 @@ defmodule Indexer.Block.Realtime.Fetcher do
end
end
+ @decorate trace(name: "fetch", resource: "Indexer.Block.Realtime.Fetcher.fetch_and_import_block/3", tracer: Tracer)
def fetch_and_import_block(block_number_to_fetch, block_fetcher, retry \\ 3) do
+ do_fetch_and_import_block(block_number_to_fetch, block_fetcher, retry)
+ end
+
+ @decorate span(tracer: Tracer)
+ defp do_fetch_and_import_block(block_number_to_fetch, block_fetcher, retry) do
case fetch_and_import_range(block_fetcher, block_number_to_fetch..block_number_to_fetch) do
{:ok, %{inserted: _, errors: []}} ->
Logger.debug(fn ->
@@ -202,7 +210,7 @@ defmodule Indexer.Block.Realtime.Fetcher do
fetcher = params.block_fetcher
updated_retry = params.retry - 1
- fetch_and_import_block(number, fetcher, updated_retry)
+ do_fetch_and_import_block(number, fetcher, updated_retry)
else
:ignore
end
diff --git a/apps/indexer/lib/indexer/block/uncle/fetcher.ex b/apps/indexer/lib/indexer/block/uncle/fetcher.ex
index 24e5499a29..07bec5c7cc 100644
--- a/apps/indexer/lib/indexer/block/uncle/fetcher.ex
+++ b/apps/indexer/lib/indexer/block/uncle/fetcher.ex
@@ -4,12 +4,14 @@ defmodule Indexer.Block.Uncle.Fetcher do
`uncle_fetched_at` where the `uncle_hash` matches `hash`.
"""
+ use Spandex.Decorators
+
require Logger
alias EthereumJSONRPC.Blocks
alias Explorer.Chain
alias Explorer.Chain.Hash
- alias Indexer.{AddressExtraction, Block, BufferedTask}
+ alias Indexer.{AddressExtraction, Block, BufferedTask, Tracer}
@behaviour Block.Fetcher
@behaviour BufferedTask
@@ -66,6 +68,7 @@ defmodule Indexer.Block.Uncle.Fetcher do
end
@impl BufferedTask
+ @decorate trace(name: "fetch", resource: "Indexer.Block.Uncle.Fetcher.run/2", service: :indexer, tracer: Tracer)
def run(hashes, %Block.Fetcher{json_rpc_named_arguments: json_rpc_named_arguments} = block_fetcher) do
# the same block could be included as an uncle on multiple blocks, but we only want to fetch it once
unique_hashes = Enum.uniq(hashes)
diff --git a/apps/indexer/lib/indexer/coin_balance/fetcher.ex b/apps/indexer/lib/indexer/coin_balance/fetcher.ex
index 27240c5aad..54a09e8139 100644
--- a/apps/indexer/lib/indexer/coin_balance/fetcher.ex
+++ b/apps/indexer/lib/indexer/coin_balance/fetcher.ex
@@ -4,6 +4,8 @@ defmodule Indexer.CoinBalance.Fetcher do
`fetched_coin_balance_block_number` to value at max `t:Explorer.Chain.Address.CoinBalance.t/0` `block_number` for the given `t:Explorer.Chain.Address.t/` `hash`.
"""
+ use Spandex.Decorators
+
require Logger
import EthereumJSONRPC, only: [integer_to_quantity: 1, quantity_to_integer: 1]
@@ -11,7 +13,7 @@ defmodule Indexer.CoinBalance.Fetcher do
alias EthereumJSONRPC.FetchedBalances
alias Explorer.Chain
alias Explorer.Chain.{Block, Hash}
- alias Indexer.BufferedTask
+ alias Indexer.{BufferedTask, Tracer}
@behaviour BufferedTask
@@ -65,6 +67,7 @@ defmodule Indexer.CoinBalance.Fetcher do
end
@impl BufferedTask
+ @decorate trace(name: "fetch", resource: "Indexer.CoinBalance.Fetcher.run/2", service: :indexer, tracer: Tracer)
def run(entries, json_rpc_named_arguments) do
# the same address may be used more than once in the same block, but we only want one `Balance` for a given
# `{address, block}`, so take unique params only
diff --git a/apps/indexer/lib/indexer/internal_transaction/fetcher.ex b/apps/indexer/lib/indexer/internal_transaction/fetcher.ex
index 65204a18ee..8af62de419 100644
--- a/apps/indexer/lib/indexer/internal_transaction/fetcher.ex
+++ b/apps/indexer/lib/indexer/internal_transaction/fetcher.ex
@@ -5,13 +5,15 @@ defmodule Indexer.InternalTransaction.Fetcher do
See `async_fetch/1` for details on configuring limits.
"""
+ use Spandex.Decorators
+
require Logger
import Indexer.Block.Fetcher, only: [async_import_coin_balances: 2]
alias Explorer.Chain
- alias Indexer.{AddressExtraction, BufferedTask}
alias Explorer.Chain.{Block, Hash}
+ alias Indexer.{AddressExtraction, BufferedTask, Tracer}
@behaviour BufferedTask
@@ -91,6 +93,12 @@ defmodule Indexer.InternalTransaction.Fetcher do
end
@impl BufferedTask
+ @decorate trace(
+ name: "fetch",
+ resource: "Indexer.InternalTransaction.Fetcher.run/2",
+ service: :indexer,
+ tracer: Tracer
+ )
def run(entries, json_rpc_named_arguments) do
unique_entries = unique_entries(entries)
diff --git a/apps/indexer/lib/indexer/token/fetcher.ex b/apps/indexer/lib/indexer/token/fetcher.ex
index eeb51dbab7..c18f1d6488 100644
--- a/apps/indexer/lib/indexer/token/fetcher.ex
+++ b/apps/indexer/lib/indexer/token/fetcher.ex
@@ -3,11 +3,13 @@ defmodule Indexer.Token.Fetcher do
Fetches information about a token.
"""
+ use Spandex.Decorators
+
alias Explorer.Chain
alias Explorer.Chain.Hash.Address
alias Explorer.Chain.Token
alias Explorer.Token.MetadataRetriever
- alias Indexer.BufferedTask
+ alias Indexer.{BufferedTask, Tracer}
@behaviour BufferedTask
@@ -47,6 +49,7 @@ defmodule Indexer.Token.Fetcher do
end
@impl BufferedTask
+ @decorate trace(name: "fetch", resource: "Indexer.Token.Fetcher.run/2", service: :indexer, tracer: Tracer)
def run([token_contract_address], _json_rpc_named_arguments) do
case Chain.token_from_address_hash(token_contract_address) do
{:ok, %Token{cataloged: false} = token} ->
diff --git a/apps/indexer/lib/indexer/token_balance/fetcher.ex b/apps/indexer/lib/indexer/token_balance/fetcher.ex
index d6acf3817c..a57d1a235b 100644
--- a/apps/indexer/lib/indexer/token_balance/fetcher.ex
+++ b/apps/indexer/lib/indexer/token_balance/fetcher.ex
@@ -14,11 +14,13 @@ defmodule Indexer.TokenBalance.Fetcher do
that always raise errors interacting with the Smart Contract.
"""
+ use Spandex.Decorators
+
require Logger
alias Explorer.Chain
alias Explorer.Chain.Hash
- alias Indexer.{BufferedTask, TokenBalances}
+ alias Indexer.{BufferedTask, TokenBalances, Tracer}
@behaviour BufferedTask
@@ -74,6 +76,7 @@ defmodule Indexer.TokenBalance.Fetcher do
when reading their balance in the Smart Contract.
"""
@impl BufferedTask
+ @decorate trace(name: "fetch", resource: "Indexer.TokenBalance.Fetcher.run/2", tracer: Tracer, service: :indexer)
def run(entries, _json_rpc_named_arguments) do
result =
entries
diff --git a/apps/indexer/lib/indexer/token_balances.ex b/apps/indexer/lib/indexer/token_balances.ex
index 481ca9b670..a00345138f 100644
--- a/apps/indexer/lib/indexer/token_balances.ex
+++ b/apps/indexer/lib/indexer/token_balances.ex
@@ -3,11 +3,14 @@ defmodule Indexer.TokenBalances do
Reads Token's balances using Smart Contract functions from the blockchain.
"""
+ use Spandex.Decorators, tracer: Indexer.Tracer
+
+ require Indexer.Tracer
require Logger
alias Explorer.Chain
alias Explorer.Token.BalanceReader
- alias Indexer.TokenBalance
+ alias Indexer.{TokenBalance, Tracer}
# The timeout used for each process opened by Task.async_stream/3. Default 15s.
@task_timeout 15000
@@ -29,14 +32,17 @@ defmodule Indexer.TokenBalances do
"""
def fetch_token_balances_from_blockchain([]), do: {:ok, []}
+ @decorate span(tracer: Tracer)
def fetch_token_balances_from_blockchain(token_balances, opts \\ []) do
Logger.debug(fn -> "fetching #{Enum.count(token_balances)} token balances" end)
task_timeout = Keyword.get(opts, :timeout, @task_timeout)
+ task_callback = traced_fetch_token_balance_callback(Tracer.current_span())
+
requested_token_balances =
token_balances
- |> Task.async_stream(&fetch_token_balance/1, timeout: task_timeout, on_timeout: :kill_task)
+ |> Task.async_stream(task_callback, timeout: task_timeout, on_timeout: :kill_task)
|> Stream.map(&format_task_results/1)
|> Enum.filter(&ignore_killed_task/1)
@@ -50,6 +56,23 @@ defmodule Indexer.TokenBalances do
{:ok, fetched_token_balances}
end
+ defp traced_fetch_token_balance_callback(%Spandex.Span{} = span) do
+ fn balance ->
+ try do
+ Tracer.continue_trace_from_span("traced_fetch_token_balance_callback/1", span)
+
+ fetch_token_balance(balance)
+ after
+ Tracer.finish_trace()
+ end
+ end
+ end
+
+ defp traced_fetch_token_balance_callback(_) do
+ &fetch_token_balance/1
+ end
+
+ @decorate span(tracer: Tracer)
defp fetch_token_balance(
%{
token_contract_address_hash: token_contract_address_hash,
diff --git a/apps/indexer/lib/indexer/tracer.ex b/apps/indexer/lib/indexer/tracer.ex
new file mode 100644
index 0000000000..7e123b99cd
--- /dev/null
+++ b/apps/indexer/lib/indexer/tracer.ex
@@ -0,0 +1,5 @@
+defmodule Indexer.Tracer do
+ @moduledoc false
+
+ use Spandex.Tracer, otp_app: :indexer
+end
diff --git a/apps/indexer/mix.exs b/apps/indexer/mix.exs
index 44b5d3dea5..ad4807a767 100644
--- a/apps/indexer/mix.exs
+++ b/apps/indexer/mix.exs
@@ -44,6 +44,8 @@ defmodule Indexer.MixProject do
# Run "mix help deps" to learn about dependencies.
defp deps do
[
+ # Optional dependency of `:spandex` for `Spandex.Decorators`
+ {:decorator, "~> 1.2"},
# JSONRPC access to Parity for `Explorer.Indexer`
{:ethereum_jsonrpc, in_umbrella: true},
# RLP encoding
@@ -57,7 +59,11 @@ defmodule Indexer.MixProject do
# Log errors and application output to separate files
{:logger_file_backend, "~> 0.0.10"},
# Mocking `EthereumJSONRPC.Transport`, so we avoid hitting real chains for local testing
- {:mox, "~> 0.4", only: [:test]}
+ {:mox, "~> 0.4", only: [:test]},
+ # Tracing
+ {:spandex, github: "spandex-project/spandex", branch: "allow-setting-trace-key", override: true},
+ # `:spandex` integration with Datadog
+ {:spandex_datadog, "~> 0.3.1"}
]
end
diff --git a/mix.exs b/mix.exs
index f76a8cfe00..70dea50bd4 100644
--- a/mix.exs
+++ b/mix.exs
@@ -60,12 +60,12 @@ defmodule BlockScout.Mixfile do
# and cannot be accessed from applications inside the apps folder
defp deps do
[
+ # Release
+ {:distillery, "~> 2.0", runtime: false},
# Documentation
{:ex_doc, "~> 0.19.0", only: [:dev]},
# Code coverage
- {:excoveralls, "~> 0.10.0", only: [:test], github: "KronicDeth/excoveralls", branch: "circle-workflows"},
- # Release
- {:distillery, "~> 2.0", runtime: false}
+ {:excoveralls, "~> 0.10.0", only: [:test], github: "KronicDeth/excoveralls", branch: "circle-workflows"}
]
end
end
diff --git a/mix.lock b/mix.lock
index 68baad2790..6d76e8744e 100644
--- a/mix.lock
+++ b/mix.lock
@@ -24,6 +24,7 @@
"dataloader": {:hex, :dataloader, "1.0.4", "7c2345c53c9e5b61420013fc53c8463ba347a938b61f66677eb47d9c4a53ac5d", [:mix], [{:ecto, ">= 0.0.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"},
"db_connection": {:hex, :db_connection, "1.1.3", "89b30ca1ef0a3b469b1c779579590688561d586694a3ce8792985d4d7e575a61", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, optional: false]}, {:poolboy, "~> 1.5", [hex: :poolboy, optional: true]}, {:sbroker, "~> 1.0", [hex: :sbroker, optional: true]}]},
"decimal": {:hex, :decimal, "1.5.0", "b0433a36d0e2430e3d50291b1c65f53c37d56f83665b43d79963684865beab68", [:mix], []},
+ "decorator": {:hex, :decorator, "1.2.4", "31dfff6143d37f0b68d0bffb3b9f18ace14fea54d4f1b5e4f86ead6f00d9ff6e", [:mix], [], "hexpm"},
"deep_merge": {:hex, :deep_merge, "0.2.0", "c1050fa2edf4848b9f556fba1b75afc66608a4219659e3311d9c9427b5b680b3", [:mix], [], "hexpm"},
"dialyxir": {:hex, :dialyxir, "0.5.1", "b331b091720fd93e878137add264bac4f644e1ddae07a70bf7062c7862c4b952", [:mix], []},
"distillery": {:hex, :distillery, "2.0.12", "6e78fe042df82610ac3fa50bd7d2d8190ad287d120d3cd1682d83a44e8b34dfb", [:mix], [{:artificery, "~> 0.2", [hex: :artificery, repo: "hexpm", optional: false]}], "hexpm"},
@@ -62,13 +63,15 @@
"math": {:hex, :math, "0.3.0", "e14e7291115201cb155a3567e66d196bf5088a6f55b030d598107d7ae934a11c", [:mix], []},
"meck": {:hex, :meck, "0.8.12", "1f7b1a9f5d12c511848fec26bbefd09a21e1432eadb8982d9a8aceb9891a3cf2", [:rebar3], [], "hexpm"},
"metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], []},
- "mime": {:hex, :mime, "1.3.0", "5e8d45a39e95c650900d03f897fbf99ae04f60ab1daa4a34c7a20a5151b7a5fe", [:mix], [], "hexpm"},
+ "mime": {:hex, :mime, "1.3.1", "30ce04ab3175b6ad0bdce0035cba77bba68b813d523d1aac73d9781b4d193cf8", [:mix], [], "hexpm"},
"mimerl": {:hex, :mimerl, "1.0.2", "993f9b0e084083405ed8252b99460c4f0563e41729ab42d9074fd5e52439be88", [:rebar3], []},
"mix_erlang_tasks": {:hex, :mix_erlang_tasks, "0.1.0", "36819fec60b80689eb1380938675af215565a89320a9e29c72c70d97512e4649", [:mix], [], "hexpm"},
"mochiweb": {:hex, :mochiweb, "2.18.0", "eb55f1db3e6e960fac4e6db4e2db9ec3602cc9f30b86cd1481d56545c3145d2e", [:rebar3], [], "hexpm"},
"mock": {:hex, :mock, "0.3.2", "e98e998fd76c191c7e1a9557c8617912c53df3d4a6132f561eb762b699ef59fa", [:mix], [{:meck, "~> 0.8.8", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm"},
"mox": {:hex, :mox, "0.4.0", "7f120840f7d626184a3d65de36189ca6f37d432e5d63acd80045198e4c5f7e6e", [:mix], [], "hexpm"},
+ "msgpax": {:hex, :msgpax, "1.1.0", "e31625e256db2decca1ae2b841f21b4d2483b1332649ce3ebc96c7ff7a4986e3", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm"},
"nimble_parsec": {:hex, :nimble_parsec, "0.4.0", "ee261bb53214943679422be70f1658fff573c5d0b0a1ecd0f18738944f818efe", [:mix], [], "hexpm"},
+ "optimal": {:hex, :optimal, "0.3.6", "46bbf52fbbbd238cda81e02560caa84f93a53c75620f1fe19e81e4ae7b07d1dd", [:mix], [], "hexpm"},
"parallel_stream": {:hex, :parallel_stream, "1.0.6", "b967be2b23f0f6787fab7ed681b4c45a215a81481fb62b01a5b750fa8f30f76c", [:mix], []},
"parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm"},
"phoenix": {:hex, :phoenix, "1.3.4", "aaa1b55e5523083a877bcbe9886d9ee180bf2c8754905323493c2ac325903dc5", [:mix], [{:cowboy, "~> 1.0", [hex: :cowboy, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.0", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.3.3 or ~> 1.4", [hex: :plug, repo: "hexpm", optional: false]}, {:poison, "~> 2.2 or ~> 3.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm"},
@@ -92,6 +95,10 @@
"ranch": {:hex, :ranch, "1.3.2", "e4965a144dc9fbe70e5c077c65e73c57165416a901bd02ea899cfd95aa890986", [:rebar3], []},
"set_locale": {:git, "https://github.com/minifast/set_locale.git", "da9ae029642bc0fbd9212c2aaf86c0adca70c084", [branch: "master"]},
"sobelow": {:hex, :sobelow, "0.7.1", "01a52ea8a19be0aa41ce969746f057e90f3994f1607c771968359718bd0e6988", [:mix], [], "hexpm"},
+ "spandex": {:git, "https://github.com/spandex-project/spandex.git", "92992b4aaf3d92e031c2417ff2e6c9e94d27fe36", [branch: "allow-setting-trace-key"]},
+ "spandex_datadog": {:hex, :spandex_datadog, "0.3.1", "984d27ad1f45cfd243509692f0f63b900a23b79566c529a644c7f3a2b4120603", [:mix], [{:msgpax, "~> 1.1", [hex: :msgpax, repo: "hexpm", optional: false]}, {:spandex, "~> 2.3", [hex: :spandex, repo: "hexpm", optional: false]}], "hexpm"},
+ "spandex_ecto": {:hex, :spandex_ecto, "0.4.0", "deaeaddc11a35f1c551206c53d09bb93a0da5808dbef751430e465c8c7de01d3", [:mix], [{:spandex, "~> 2.2", [hex: :spandex, repo: "hexpm", optional: false]}], "hexpm"},
+ "spandex_phoenix": {:hex, :spandex_phoenix, "0.3.0", "48b0a426bbe4eea3e579bbea77d5eb5a8d4b83d33c95616f9ba64b3ce2faef6c", [:mix], [{:plug, "~> 1.3", [hex: :plug, repo: "hexpm", optional: false]}, {:spandex, "~> 2.2", [hex: :spandex, repo: "hexpm", optional: false]}], "hexpm"},
"ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"},
"timex": {:hex, :timex, "3.4.1", "e63fc1a37453035e534c3febfe9b6b9e18583ec7b37fd9c390efdef97397d70b", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"},
"timex_ecto": {:hex, :timex_ecto, "3.3.0", "d5bdef09928e7a60f10a0baa47ce653f29b43d6fee87b30b236b216d0e36b98d", [:mix], [{:ecto, "~> 2.2", [hex: :ecto, repo: "hexpm", optional: false]}, {:timex, "~> 3.1", [hex: :timex, repo: "hexpm", optional: false]}], "hexpm"},