Merge pull request #1150 from poanetwork/support-tracing-via-spandex

feat: support tracing via spandex

.credo.exs

@@ -75,8 +75,9 @@
         # Priority values are: `low, normal, high, higher`
         #
         {Credo.Check.Design.AliasUsage,
-         excluded_namespaces: ~w(Block Blocks Import Socket Task),
+         excluded_namespaces: ~w(Block Blocks Import Socket SpandexDatadog Task),
-         excluded_lastnames: ~w(Address DateTime Exporter Fetcher Full Instrumenter Monitor Name Number Repo Time Unit),
+         excluded_lastnames:
+           ~w(Address DateTime Exporter Fetcher Full Instrumenter Monitor Name Number Repo Spec Time Unit),
          priority: :low},
 
         # For some checks, you can also set other parameters

README.md

@@ -7,7 +7,7 @@
 <h1 align="center">BlockScout</h1>
 <p align="center">Blockchain Explorer for inspecting and analyzing EVM Chains.</p>
 <div align="center">
-  
+
 [![CircleCI](https://circleci.com/gh/poanetwork/blockscout.svg?style=svg&circle-token=f8823a3d0090407c11f87028c73015a331dbf604)](https://circleci.com/gh/poanetwork/blockscout) [![Coverage Status](https://coveralls.io/repos/github/poanetwork/blockscout/badge.svg?branch=master)](https://coveralls.io/github/poanetwork/blockscout?branch=master) [![Join the chat at https://gitter.im/poanetwork/blockscout](https://badges.gitter.im/poanetwork/blockscout.svg)](https://gitter.im/poanetwork/blockscout?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
 </div>
@@ -283,6 +283,33 @@ BlockScout is setup to export [Prometheus](https://prometheus.io/) metrics at `/
    3. Click "Load"
 6. View the dashboards.  (You will need to click-around and use BlockScout for the web-related metrics to show up.)
 
+## Tracing
+
+Blockscout supports tracing via
+[Spandex](http://git@github.com:spandex-project/spandex.git). Each application
+has its own tracer, that is configured internally to that application. In order
+to enable it, visit each application's `config/<env>.ex` and update its tracer
+configuration to change `disabled?: true` to `disabled?: false`. Do this for
+each application you'd like included in your trace data.
+
+Currently, only [Datadog](https://www.datadoghq.com/) is supported as a
+tracing backend, but more will be added soon.
+
+### DataDog
+
+If you would like to use DataDog, after enabling `Spandex`, set
+`"DATADOG_HOST"` and `"DATADOG_PORT"` environment variables to the
+host/port that your Datadog agent is running on. For more information on
+Datadog and the Datadog agent, see their
+[documentation](https://docs.datadoghq.com/).
+
+### Other
+
+If you want to use a different  backend, remove the
+`SpandexDatadog.ApiServer` `Supervisor.child_spec` from
+`Explorer.Application` and follow any instructions provided in `Spandex`
+for setting up that backend.
+
 ## Memory Usage
 
 The work queues for building the index of all blocks, balances (coin and token), and internal transactions can grow quite large.   By default, the soft-limit is 1 GiB, which can be changed in `apps/indexer/config/config.exs`:

apps/block_scout_web/config/config.exs

@@ -18,7 +18,7 @@ config :block_scout_web, BlockScoutWeb.Chain,
 
 # Configures the endpoint
 config :block_scout_web, BlockScoutWeb.Endpoint,
-  instrumenters: [BlockScoutWeb.Prometheus.Instrumenter],
+  instrumenters: [BlockScoutWeb.Prometheus.Instrumenter, SpandexPhoenix.Instrumenter],
   url: [
     host: "localhost",
     path: System.get_env("NETWORK_PATH") || "/"
@@ -26,6 +26,11 @@ config :block_scout_web, BlockScoutWeb.Endpoint,
   render_errors: [view: BlockScoutWeb.ErrorView, accepts: ~w(html json)],
   pubsub: [name: BlockScoutWeb.PubSub, adapter: Phoenix.PubSub.PG2]
 
+config :block_scout_web, BlockScoutWeb.Tracer,
+  service: :block_scout_web,
+  adapter: SpandexDatadog.Adapter,
+  trace_key: :blockscout
+
 # Configures gettext
 config :block_scout_web, BlockScoutWeb.Gettext, locales: ~w(en), default_locale: "en"
 
@@ -46,6 +51,8 @@ config :logger, :block_scout_web,
   metadata: [:application, :request_id],
   metadata_filter: [application: :block_scout_web]
 
+config :spandex_phoenix, tracer: BlockScoutWeb.Tracer
+
 config :wobserver,
   # return only the local node
   discovery: :none,

apps/block_scout_web/config/dev.exs

@@ -48,6 +48,8 @@ config :block_scout_web, BlockScoutWeb.Endpoint,
     ]
   ]
 
+config :block_scout_web, BlockScoutWeb.Tracer, env: "dev", disabled?: true
+
 config :logger, :block_scout_web,
   level: :debug,
   path: Path.absname("logs/dev/block_scout_web.log")

apps/block_scout_web/config/prod.exs

@@ -24,6 +24,8 @@ config :block_scout_web, BlockScoutWeb.Endpoint,
     port: System.get_env("PORT")
   ]
 
+config :block_scout_web, BlockScoutWeb.Tracer, env: "production", disabled?: true
+
 config :logger, :block_scout_web,
   level: :info,
   path: Path.absname("logs/prod/block_scout_web.log"),

apps/block_scout_web/config/test.exs

@@ -9,6 +9,8 @@ config :block_scout_web, BlockScoutWeb.Endpoint,
   secret_key_base: "27Swe6KtEtmN37WyEYRjKWyxYULNtrxlkCEKur4qoV+Lwtk8lafsR16ifz1XBBYj",
   server: true
 
+config :block_scout_web, BlockScoutWeb.Tracer, disabled?: false
+
 config :logger, :block_scout_web,
   level: :warn,
   path: Path.absname("logs/test/block_scout_web.log")

apps/block_scout_web/lib/block_scout_web/endpoint.ex

@@ -68,6 +68,8 @@ defmodule BlockScoutWeb.Endpoint do
     signing_salt: "iC2ksJHS"
   )
 
+  use SpandexPhoenix
+
   plug(BlockScoutWeb.Prometheus.Exporter)
 
   plug(BlockScoutWeb.Router)

apps/block_scout_web/lib/block_scout_web/tracer.ex

@@ -0,0 +1,5 @@
+defmodule BlockScoutWeb.Tracer do
+  @moduledoc false
+
+  use Spandex.Tracer, otp_app: :block_scout_web
+end

apps/block_scout_web/mix.exs

@@ -112,6 +112,12 @@ defmodule BlockScoutWeb.Mixfile do
       {:prometheus_process_collector, "~> 1.3"},
       {:qrcode, "~> 0.1.0"},
       {:sobelow, ">= 0.7.0", only: [:dev, :test], runtime: false},
+      # Tracing
+      {:spandex, github: "spandex-project/spandex", branch: "allow-setting-trace-key", override: true},
+      # `:spandex` integration with Datadog
+      {:spandex_datadog, "~> 0.3.1"},
+      # `:spandex` tracing of `:phoenix`
+      {:spandex_phoenix, "~> 0.3.0"},
       {:timex, "~> 3.4"},
       {:wallaby, "~> 0.20", only: [:test], runtime: false},
       {:wobserver, "~> 0.1.8"}

apps/ethereum_jsonrpc/config/config.exs

@@ -1,11 +1,5 @@
 use Mix.Config
 
-config :logger, :ethereum_jsonrpc,
-  # keep synced with `config/config.exs`
-  format: "$time $metadata[$level] $message\n",
-  metadata: [:application, :request_id],
-  metadata_filter: [application: :ethereum_jsonrpc]
-
 config :ethereum_jsonrpc, EthereumJSONRPC.RequestCoordinator,
   rolling_window_opts: [
     window_count: 12,
@@ -15,6 +9,17 @@ config :ethereum_jsonrpc, EthereumJSONRPC.RequestCoordinator,
   wait_per_timeout: :timer.seconds(20),
   max_jitter: :timer.seconds(2)
 
+config :ethereum_jsonrpc, EthereumJSONRPC.Tracer,
+  service: :ethereum_jsonrpc,
+  adapter: SpandexDatadog.Adapter,
+  trace_key: :blockscout
+
+config :logger, :ethereum_jsonrpc,
+  # keep synced with `config/config.exs`
+  format: "$time $metadata[$level] $message\n",
+  metadata: [:application, :request_id],
+  metadata_filter: [application: :ethereum_jsonrpc]
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"

apps/ethereum_jsonrpc/config/dev.exs

@@ -1,5 +1,7 @@
 use Mix.Config
 
+config :ethereum_jsonrpc, EthereumJSONRPC.Tracer, env: "dev", disabled?: true
+
 config :logger, :ethereum_jsonrpc,
   level: :debug,
   path: Path.absname("logs/dev/ethereum_jsonrpc.log")

apps/ethereum_jsonrpc/config/prod.exs

@@ -1,5 +1,7 @@
 use Mix.Config
 
+config :ethereum_jsonrpc, EthereumJSONRPC.Tracer, env: "production", disabled?: true
+
 config :logger, :ethereum_jsonrpc,
   level: :info,
   path: Path.absname("logs/prod/ethereum_jsonrpc.log"),

apps/ethereum_jsonrpc/config/test.exs

@@ -1,9 +1,5 @@
 use Mix.Config
 
-config :logger, :ethereum_jsonrpc,
-  level: :warn,
-  path: Path.absname("logs/test/ethereum_jsonrpc.log")
-
 config :ethereum_jsonrpc, EthereumJSONRPC.RequestCoordinator,
   rolling_window_opts: [
     window_count: 3,
@@ -12,3 +8,9 @@ config :ethereum_jsonrpc, EthereumJSONRPC.RequestCoordinator,
   ],
   wait_per_timeout: 2,
   max_jitter: 1
+
+config :ethereum_jsonrpc, EthereumJSONRPC.Tracer, disabled?: false
+
+config :logger, :ethereum_jsonrpc,
+  level: :warn,
+  path: Path.absname("logs/test/ethereum_jsonrpc.log")

apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/request_coordinator.ex

@@ -42,7 +42,9 @@ defmodule EthereumJSONRPC.RequestCoordinator do
   With this configuration, timeouts are tracked for 6 windows of 10 seconds for a total of 1 minute.
   """
 
-  alias EthereumJSONRPC.{RollingWindow, Transport}
+  require EthereumJSONRPC.Tracer
+
+  alias EthereumJSONRPC.{RollingWindow, Tracer, Transport}
 
   @error_key :throttleable_error_count
 
@@ -63,9 +65,11 @@ defmodule EthereumJSONRPC.RequestCoordinator do
     if sleep_time <= throttle_timeout do
       :timer.sleep(sleep_time)
 
-      request
+      trace_request(request, fn ->
-      |> transport.json_rpc(transport_options)
+        request
-      |> handle_transport_response()
+        |> transport.json_rpc(transport_options)
+        |> handle_transport_response()
+      end)
     else
       :timer.sleep(throttle_timeout)
 
@@ -73,6 +77,18 @@ defmodule EthereumJSONRPC.RequestCoordinator do
     end
   end
 
+  defp trace_request([request | _], fun) do
+    trace_request(request, fun)
+  end
+
+  defp trace_request(%{method: method}, fun) do
+    Tracer.span "RequestCoordinator.perform/4", resource: method, service: :ethereum_jsonrpc do
+      fun.()
+    end
+  end
+
+  defp trace_request(_, fun), do: fun.()
+
   defp handle_transport_response({:error, {:bad_gateway, _}} = error) do
     RollingWindow.inc(table(), @error_key)
     error

apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/tracer.ex

@@ -0,0 +1,5 @@
+defmodule EthereumJSONRPC.Tracer do
+  @moduledoc false
+
+  use Spandex.Tracer, otp_app: :ethereum_jsonrpc
+end

apps/ethereum_jsonrpc/mix.exs

@@ -76,6 +76,10 @@ defmodule EthereumJsonrpc.MixProject do
       {:logger_file_backend, "~> 0.0.10"},
       # Mocking `EthereumJSONRPC.Transport` and `EthereumJSONRPC.HTTP` so we avoid hitting real chains for local testing
       {:mox, "~> 0.4", only: [:test]},
+      # Tracing
+      {:spandex, github: "spandex-project/spandex", branch: "allow-setting-trace-key", override: true},
+      # `:spandex` integration with Datadog
+      {:spandex_datadog, "~> 0.3.1"},
       # Convert unix timestamps in JSONRPC to DateTimes
       {:timex, "~> 3.4"},
       # Encode/decode function names and arguments

apps/explorer/config/config.exs

@@ -25,6 +25,11 @@ config :explorer, Explorer.Repo,
   loggers: [Explorer.Repo.PrometheusLogger, Ecto.LogEntry],
   migration_timestamps: [type: :utc_datetime]
 
+config :explorer, Explorer.Tracer,
+  service: :explorer,
+  adapter: SpandexDatadog.Adapter,
+  trace_key: :blockscout
+
 config :explorer, Explorer.Counters.TokenTransferCounter, enabled: true
 
 config :explorer, Explorer.Counters.TokenHoldersCounter, enabled: true, enable_consolidation: true
@@ -46,6 +51,11 @@ config :logger, :explorer,
   metadata: [:application, :request_id],
   metadata_filter: [application: :explorer]
 
+config :spandex_ecto, SpandexEcto.EctoLogger,
+  service: :ecto,
+  tracer: Explorer.Tracer,
+  otp_app: :explorer
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"

apps/explorer/config/dev.exs

@@ -9,6 +9,8 @@ config :explorer, Explorer.Repo,
   pool_timeout: 60_000,
   timeout: 80_000
 
+config :explorer, Explorer.Tracer, env: "dev", disabled?: true
+
 config :logger, :explorer,
   level: :debug,
   path: Path.absname("logs/dev/explorer.log")

apps/explorer/config/prod.exs

@@ -9,6 +9,8 @@ config :explorer, Explorer.Repo,
   prepare: :unnamed,
   timeout: 60_000
 
+config :explorer, Explorer.Tracer, env: "production", disabled?: true
+
 config :logger, :explorer,
   level: :info,
   path: Path.absname("logs/prod/explorer.log"),

apps/explorer/config/test.exs

@@ -3,6 +3,8 @@ use Mix.Config
 # Lower hashing rounds for faster tests
 config :bcrypt_elixir, log_rounds: 4
 
+config :explorer, Explorer.Counters.TokenHoldersCounter, enabled: true, enable_consolidation: false
+
 # Configure your database
 config :explorer, Explorer.Repo,
   adapter: Ecto.Adapters.Postgres,
@@ -17,7 +19,7 @@ config :explorer, Explorer.ExchangeRates, enabled: false
 
 config :explorer, Explorer.Market.History.Cataloger, enabled: false
 
-config :explorer, Explorer.Counters.TokenHoldersCounter, enabled: true, enable_consolidation: false
+config :explorer, Explorer.Tracer, disabled?: false
 
 config :logger, :explorer,
   level: :warn,

apps/explorer/lib/explorer/application.ex

@@ -15,6 +15,7 @@ defmodule Explorer.Application do
     # Children to start in all environments
     base_children = [
       Explorer.Repo,
+      Supervisor.Spec.worker(SpandexDatadog.ApiServer, [datadog_opts()]),
       Supervisor.child_spec({Task.Supervisor, name: Explorer.MarketTaskSupervisor}, id: Explorer.MarketTaskSupervisor),
       Supervisor.child_spec({Task.Supervisor, name: Explorer.TaskSupervisor}, id: Explorer.TaskSupervisor),
       {Registry, keys: :duplicate, name: Registry.ChainEvents, id: Registry.ChainEvents},
@@ -52,4 +53,14 @@ defmodule Explorer.Application do
       []
     end
   end
+
+  defp datadog_opts do
+    [
+      host: System.get_env("DATADOG_HOST") || "localhost",
+      port: System.get_env("DATADOG_PORT") || 8126,
+      batch_size: System.get_env("SPANDEX_BATCH_SIZE") || 100,
+      sync_threshold: System.get_env("SPANDEX_SYNC_THRESHOLD") || 100,
+      http: HTTPoison
+    ]
+  end
 end

apps/explorer/lib/explorer/tracer.ex

@@ -0,0 +1,5 @@
+defmodule Explorer.Tracer do
+  @moduledoc false
+
+  use Spandex.Tracer, otp_app: :explorer
+end

apps/explorer/mix.exs

@@ -96,6 +96,12 @@ defmodule Explorer.Mixfile do
       # bypass optional dependency
       {:plug_cowboy, "~> 1.0", only: :test},
       {:sobelow, ">= 0.7.0", only: [:dev, :test], runtime: false},
+      # Tracing
+      {:spandex, github: "spandex-project/spandex", branch: "allow-setting-trace-key", override: true},
+      # `:spandex` integration with Datadog
+      {:spandex_datadog, "~> 0.3.1"},
+      # `:spandex` tracing of `:ecto`
+      {:spandex_ecto, "~> 0.4.0"},
       {:timex, "~> 3.4"},
       # `Timex.Duration` for `Explorer.Chain.average_block_time/0`
       {:timex_ecto, "~> 3.3"}

apps/indexer/config/config.exs

@@ -11,6 +11,11 @@ config :indexer,
   # bytes
   memory_limit: 1 <<< 30
 
+config :indexer, Indexer.Tracer,
+  service: :indexer,
+  adapter: SpandexDatadog.Adapter,
+  trace_key: :blockscout
+
 config :logger, :indexer,
   # keep synced with `config/config.exs`
   format: "$time $metadata[$level] $message\n",

apps/indexer/config/dev.exs

@@ -1,5 +1,7 @@
 use Mix.Config
 
+config :indexer, Indexer.Tracer, env: "dev", disabled?: true
+
 config :logger, :indexer,
   level: :debug,
   path: Path.absname("logs/dev/indexer.log")

apps/indexer/config/prod.exs

@@ -1,5 +1,7 @@
 use Mix.Config
 
+config :indexer, Indexer.Tracer, env: "production", disabled?: true
+
 config :logger, :indexer,
   level: :info,
   path: Path.absname("logs/prod/indexer.log"),

apps/indexer/config/test.exs

@@ -1,5 +1,7 @@
 use Mix.Config
 
+config :indexer, Indexer.Tracer, disabled?: false
+
 config :logger, :indexer,
   level: :warn,
   path: Path.absname("logs/test/indexer.log")

apps/indexer/lib/indexer/block/catchup/fetcher.ex

@@ -3,13 +3,15 @@ defmodule Indexer.Block.Catchup.Fetcher do
   Fetches and indexes block ranges from the block before the latest block to genesis (0) that are missing.
   """
 
+  use Spandex.Decorators
+
   require Logger
 
   import Indexer.Block.Fetcher,
     only: [async_import_coin_balances: 2, async_import_tokens: 1, async_import_uncles: 1, fetch_and_import_range: 2]
 
   alias Explorer.Chain
-  alias Indexer.{Block, InternalTransaction, Sequence, TokenBalance}
+  alias Indexer.{Block, InternalTransaction, Sequence, TokenBalance, Tracer}
   alias Indexer.Memory.Shrinkable
 
   @behaviour Block.Fetcher
@@ -160,6 +162,11 @@ defmodule Indexer.Block.Catchup.Fetcher do
   end
 
   # Run at state.blocks_concurrency max_concurrency when called by `stream_import/1`
+  @decorate trace(
+              name: "fetch",
+              resource: "Indexer.Block.Catchup.Fetcher.fetch_and_import_range_from_sequence/3",
+              tracer: Tracer
+            )
   defp fetch_and_import_range_from_sequence(
          %__MODULE__{block_fetcher: %Block.Fetcher{} = block_fetcher},
          _.._ = range,

apps/indexer/lib/indexer/block/fetcher.ex

@@ -3,11 +3,13 @@ defmodule Indexer.Block.Fetcher do
   Fetches and indexes block ranges.
   """
 
+  use Spandex.Decorators
+
   require Logger
 
   alias EthereumJSONRPC.{Blocks, FetchedBeneficiaries}
   alias Explorer.Chain.{Address, Block, Import}
-  alias Indexer.{AddressExtraction, CoinBalance, MintTransfer, Token, TokenTransfers}
+  alias Indexer.{AddressExtraction, CoinBalance, MintTransfer, Token, TokenTransfers, Tracer}
   alias Indexer.Address.{CoinBalances, TokenBalances}
   alias Indexer.Block.Fetcher.Receipts
   alias Indexer.Block.Transform
@@ -76,6 +78,7 @@ defmodule Indexer.Block.Fetcher do
     struct!(__MODULE__, named_arguments)
   end
 
+  @decorate span(tracer: Tracer)
   @spec fetch_and_import_range(t, Range.t()) ::
           {:ok, %{inserted: %{}, errors: [EthereumJSONRPC.Transport.error()]}}
           | {:error,

apps/indexer/lib/indexer/block/realtime/fetcher.ex

@@ -4,7 +4,9 @@ defmodule Indexer.Block.Realtime.Fetcher do
   """
 
   use GenServer
+  use Spandex.Decorators
 
+  require Indexer.Tracer
   require Logger
 
   import EthereumJSONRPC, only: [integer_to_quantity: 1, quantity_to_integer: 1]
@@ -13,7 +15,7 @@ defmodule Indexer.Block.Realtime.Fetcher do
   alias Ecto.Changeset
   alias EthereumJSONRPC.{FetchedBalances, Subscription}
   alias Explorer.Chain
-  alias Indexer.{AddressExtraction, Block, TokenBalances}
+  alias Indexer.{AddressExtraction, Block, TokenBalances, Tracer}
   alias Indexer.Block.Realtime.TaskSupervisor
 
   @behaviour Block.Fetcher
@@ -125,7 +127,13 @@ defmodule Indexer.Block.Realtime.Fetcher do
     end
   end
 
+  @decorate trace(name: "fetch", resource: "Indexer.Block.Realtime.Fetcher.fetch_and_import_block/3", tracer: Tracer)
   def fetch_and_import_block(block_number_to_fetch, block_fetcher, retry \\ 3) do
+    do_fetch_and_import_block(block_number_to_fetch, block_fetcher, retry)
+  end
+
+  @decorate span(tracer: Tracer)
+  defp do_fetch_and_import_block(block_number_to_fetch, block_fetcher, retry) do
     case fetch_and_import_range(block_fetcher, block_number_to_fetch..block_number_to_fetch) do
       {:ok, %{inserted: _, errors: []}} ->
         Logger.debug(fn ->
@@ -202,7 +210,7 @@ defmodule Indexer.Block.Realtime.Fetcher do
       fetcher = params.block_fetcher
       updated_retry = params.retry - 1
 
-      fetch_and_import_block(number, fetcher, updated_retry)
+      do_fetch_and_import_block(number, fetcher, updated_retry)
     else
       :ignore
     end

apps/indexer/lib/indexer/block/uncle/fetcher.ex

@@ -4,12 +4,14 @@ defmodule Indexer.Block.Uncle.Fetcher do
   `uncle_fetched_at` where the `uncle_hash` matches `hash`.
   """
 
+  use Spandex.Decorators
+
   require Logger
 
   alias EthereumJSONRPC.Blocks
   alias Explorer.Chain
   alias Explorer.Chain.Hash
-  alias Indexer.{AddressExtraction, Block, BufferedTask}
+  alias Indexer.{AddressExtraction, Block, BufferedTask, Tracer}
 
   @behaviour Block.Fetcher
   @behaviour BufferedTask
@@ -66,6 +68,7 @@ defmodule Indexer.Block.Uncle.Fetcher do
   end
 
   @impl BufferedTask
+  @decorate trace(name: "fetch", resource: "Indexer.Block.Uncle.Fetcher.run/2", service: :indexer, tracer: Tracer)
   def run(hashes, %Block.Fetcher{json_rpc_named_arguments: json_rpc_named_arguments} = block_fetcher) do
     # the same block could be included as an uncle on multiple blocks, but we only want to fetch it once
     unique_hashes = Enum.uniq(hashes)

apps/indexer/lib/indexer/coin_balance/fetcher.ex

@@ -4,6 +4,8 @@ defmodule Indexer.CoinBalance.Fetcher do
   `fetched_coin_balance_block_number` to value at max `t:Explorer.Chain.Address.CoinBalance.t/0` `block_number` for the given `t:Explorer.Chain.Address.t/` `hash`.
   """
 
+  use Spandex.Decorators
+
   require Logger
 
   import EthereumJSONRPC, only: [integer_to_quantity: 1, quantity_to_integer: 1]
@@ -11,7 +13,7 @@ defmodule Indexer.CoinBalance.Fetcher do
   alias EthereumJSONRPC.FetchedBalances
   alias Explorer.Chain
   alias Explorer.Chain.{Block, Hash}
-  alias Indexer.BufferedTask
+  alias Indexer.{BufferedTask, Tracer}
 
   @behaviour BufferedTask
 
@@ -65,6 +67,7 @@ defmodule Indexer.CoinBalance.Fetcher do
   end
 
   @impl BufferedTask
+  @decorate trace(name: "fetch", resource: "Indexer.CoinBalance.Fetcher.run/2", service: :indexer, tracer: Tracer)
   def run(entries, json_rpc_named_arguments) do
     # the same address may be used more than once in the same block, but we only want one `Balance` for a given
     # `{address, block}`, so take unique params only

apps/indexer/lib/indexer/internal_transaction/fetcher.ex

@@ -5,13 +5,15 @@ defmodule Indexer.InternalTransaction.Fetcher do
   See `async_fetch/1` for details on configuring limits.
   """
 
+  use Spandex.Decorators
+
   require Logger
 
   import Indexer.Block.Fetcher, only: [async_import_coin_balances: 2]
 
   alias Explorer.Chain
-  alias Indexer.{AddressExtraction, BufferedTask}
   alias Explorer.Chain.{Block, Hash}
+  alias Indexer.{AddressExtraction, BufferedTask, Tracer}
 
   @behaviour BufferedTask
 
@@ -91,6 +93,12 @@ defmodule Indexer.InternalTransaction.Fetcher do
   end
 
   @impl BufferedTask
+  @decorate trace(
+              name: "fetch",
+              resource: "Indexer.InternalTransaction.Fetcher.run/2",
+              service: :indexer,
+              tracer: Tracer
+            )
   def run(entries, json_rpc_named_arguments) do
     unique_entries = unique_entries(entries)
 

apps/indexer/lib/indexer/token/fetcher.ex

@@ -3,11 +3,13 @@ defmodule Indexer.Token.Fetcher do
   Fetches information about a token.
   """
 
+  use Spandex.Decorators
+
   alias Explorer.Chain
   alias Explorer.Chain.Hash.Address
   alias Explorer.Chain.Token
   alias Explorer.Token.MetadataRetriever
-  alias Indexer.BufferedTask
+  alias Indexer.{BufferedTask, Tracer}
 
   @behaviour BufferedTask
 
@@ -47,6 +49,7 @@ defmodule Indexer.Token.Fetcher do
   end
 
   @impl BufferedTask
+  @decorate trace(name: "fetch", resource: "Indexer.Token.Fetcher.run/2", service: :indexer, tracer: Tracer)
   def run([token_contract_address], _json_rpc_named_arguments) do
     case Chain.token_from_address_hash(token_contract_address) do
       {:ok, %Token{cataloged: false} = token} ->

apps/indexer/lib/indexer/token_balance/fetcher.ex

@@ -14,11 +14,13 @@ defmodule Indexer.TokenBalance.Fetcher do
   that always raise errors interacting with the Smart Contract.
   """
 
+  use Spandex.Decorators
+
   require Logger
 
   alias Explorer.Chain
   alias Explorer.Chain.Hash
-  alias Indexer.{BufferedTask, TokenBalances}
+  alias Indexer.{BufferedTask, TokenBalances, Tracer}
 
   @behaviour BufferedTask
 
@@ -74,6 +76,7 @@ defmodule Indexer.TokenBalance.Fetcher do
   when reading their balance in the Smart Contract.
   """
   @impl BufferedTask
+  @decorate trace(name: "fetch", resource: "Indexer.TokenBalance.Fetcher.run/2", tracer: Tracer, service: :indexer)
   def run(entries, _json_rpc_named_arguments) do
     result =
       entries

apps/indexer/lib/indexer/token_balances.ex

@@ -3,11 +3,14 @@ defmodule Indexer.TokenBalances do
   Reads Token's balances using Smart Contract functions from the blockchain.
   """
 
+  use Spandex.Decorators, tracer: Indexer.Tracer
+
+  require Indexer.Tracer
   require Logger
 
   alias Explorer.Chain
   alias Explorer.Token.BalanceReader
-  alias Indexer.TokenBalance
+  alias Indexer.{TokenBalance, Tracer}
 
   # The timeout used for each process opened by Task.async_stream/3. Default 15s.
   @task_timeout 15000
@@ -29,14 +32,17 @@ defmodule Indexer.TokenBalances do
   """
   def fetch_token_balances_from_blockchain([]), do: {:ok, []}
 
+  @decorate span(tracer: Tracer)
   def fetch_token_balances_from_blockchain(token_balances, opts \\ []) do
     Logger.debug(fn -> "fetching #{Enum.count(token_balances)} token balances" end)
 
     task_timeout = Keyword.get(opts, :timeout, @task_timeout)
 
+    task_callback = traced_fetch_token_balance_callback(Tracer.current_span())
+
     requested_token_balances =
       token_balances
-      |> Task.async_stream(&fetch_token_balance/1, timeout: task_timeout, on_timeout: :kill_task)
+      |> Task.async_stream(task_callback, timeout: task_timeout, on_timeout: :kill_task)
       |> Stream.map(&format_task_results/1)
       |> Enum.filter(&ignore_killed_task/1)
 
@@ -50,6 +56,23 @@ defmodule Indexer.TokenBalances do
     {:ok, fetched_token_balances}
   end
 
+  defp traced_fetch_token_balance_callback(%Spandex.Span{} = span) do
+    fn balance ->
+      try do
+        Tracer.continue_trace_from_span("traced_fetch_token_balance_callback/1", span)
+
+        fetch_token_balance(balance)
+      after
+        Tracer.finish_trace()
+      end
+    end
+  end
+
+  defp traced_fetch_token_balance_callback(_) do
+    &fetch_token_balance/1
+  end
+
+  @decorate span(tracer: Tracer)
   defp fetch_token_balance(
          %{
            token_contract_address_hash: token_contract_address_hash,

apps/indexer/lib/indexer/tracer.ex

@@ -0,0 +1,5 @@
+defmodule Indexer.Tracer do
+  @moduledoc false
+
+  use Spandex.Tracer, otp_app: :indexer
+end

apps/indexer/mix.exs

@@ -44,6 +44,8 @@ defmodule Indexer.MixProject do
   # Run "mix help deps" to learn about dependencies.
   defp deps do
     [
+      # Optional dependency of `:spandex` for `Spandex.Decorators`
+      {:decorator, "~> 1.2"},
       # JSONRPC access to Parity for `Explorer.Indexer`
       {:ethereum_jsonrpc, in_umbrella: true},
       # RLP encoding
@@ -57,7 +59,11 @@ defmodule Indexer.MixProject do
       # Log errors and application output to separate files
       {:logger_file_backend, "~> 0.0.10"},
       # Mocking `EthereumJSONRPC.Transport`, so we avoid hitting real chains for local testing
-      {:mox, "~> 0.4", only: [:test]}
+      {:mox, "~> 0.4", only: [:test]},
+      # Tracing
+      {:spandex, github: "spandex-project/spandex", branch: "allow-setting-trace-key", override: true},
+      # `:spandex` integration with Datadog
+      {:spandex_datadog, "~> 0.3.1"}
     ]
   end
 

mix.exs

@@ -60,12 +60,12 @@ defmodule BlockScout.Mixfile do
   # and cannot be accessed from applications inside the apps folder
   defp deps do
     [
+      # Release
+      {:distillery, "~> 2.0", runtime: false},
       # Documentation
       {:ex_doc, "~> 0.19.0", only: [:dev]},
       # Code coverage
-      {:excoveralls, "~> 0.10.0", only: [:test], github: "KronicDeth/excoveralls", branch: "circle-workflows"},
+      {:excoveralls, "~> 0.10.0", only: [:test], github: "KronicDeth/excoveralls", branch: "circle-workflows"}
-      # Release
-      {:distillery, "~> 2.0", runtime: false}
     ]
   end
 end

mix.lock

@@ -24,6 +24,7 @@
   "dataloader": {:hex, :dataloader, "1.0.4", "7c2345c53c9e5b61420013fc53c8463ba347a938b61f66677eb47d9c4a53ac5d", [:mix], [{:ecto, ">= 0.0.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"},
   "db_connection": {:hex, :db_connection, "1.1.3", "89b30ca1ef0a3b469b1c779579590688561d586694a3ce8792985d4d7e575a61", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, optional: false]}, {:poolboy, "~> 1.5", [hex: :poolboy, optional: true]}, {:sbroker, "~> 1.0", [hex: :sbroker, optional: true]}]},
   "decimal": {:hex, :decimal, "1.5.0", "b0433a36d0e2430e3d50291b1c65f53c37d56f83665b43d79963684865beab68", [:mix], []},
+  "decorator": {:hex, :decorator, "1.2.4", "31dfff6143d37f0b68d0bffb3b9f18ace14fea54d4f1b5e4f86ead6f00d9ff6e", [:mix], [], "hexpm"},
   "deep_merge": {:hex, :deep_merge, "0.2.0", "c1050fa2edf4848b9f556fba1b75afc66608a4219659e3311d9c9427b5b680b3", [:mix], [], "hexpm"},
   "dialyxir": {:hex, :dialyxir, "0.5.1", "b331b091720fd93e878137add264bac4f644e1ddae07a70bf7062c7862c4b952", [:mix], []},
   "distillery": {:hex, :distillery, "2.0.12", "6e78fe042df82610ac3fa50bd7d2d8190ad287d120d3cd1682d83a44e8b34dfb", [:mix], [{:artificery, "~> 0.2", [hex: :artificery, repo: "hexpm", optional: false]}], "hexpm"},
@@ -62,13 +63,15 @@
   "math": {:hex, :math, "0.3.0", "e14e7291115201cb155a3567e66d196bf5088a6f55b030d598107d7ae934a11c", [:mix], []},
   "meck": {:hex, :meck, "0.8.12", "1f7b1a9f5d12c511848fec26bbefd09a21e1432eadb8982d9a8aceb9891a3cf2", [:rebar3], [], "hexpm"},
   "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], []},
-  "mime": {:hex, :mime, "1.3.0", "5e8d45a39e95c650900d03f897fbf99ae04f60ab1daa4a34c7a20a5151b7a5fe", [:mix], [], "hexpm"},
+  "mime": {:hex, :mime, "1.3.1", "30ce04ab3175b6ad0bdce0035cba77bba68b813d523d1aac73d9781b4d193cf8", [:mix], [], "hexpm"},
   "mimerl": {:hex, :mimerl, "1.0.2", "993f9b0e084083405ed8252b99460c4f0563e41729ab42d9074fd5e52439be88", [:rebar3], []},
   "mix_erlang_tasks": {:hex, :mix_erlang_tasks, "0.1.0", "36819fec60b80689eb1380938675af215565a89320a9e29c72c70d97512e4649", [:mix], [], "hexpm"},
   "mochiweb": {:hex, :mochiweb, "2.18.0", "eb55f1db3e6e960fac4e6db4e2db9ec3602cc9f30b86cd1481d56545c3145d2e", [:rebar3], [], "hexpm"},
   "mock": {:hex, :mock, "0.3.2", "e98e998fd76c191c7e1a9557c8617912c53df3d4a6132f561eb762b699ef59fa", [:mix], [{:meck, "~> 0.8.8", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm"},
   "mox": {:hex, :mox, "0.4.0", "7f120840f7d626184a3d65de36189ca6f37d432e5d63acd80045198e4c5f7e6e", [:mix], [], "hexpm"},
+  "msgpax": {:hex, :msgpax, "1.1.0", "e31625e256db2decca1ae2b841f21b4d2483b1332649ce3ebc96c7ff7a4986e3", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm"},
   "nimble_parsec": {:hex, :nimble_parsec, "0.4.0", "ee261bb53214943679422be70f1658fff573c5d0b0a1ecd0f18738944f818efe", [:mix], [], "hexpm"},
+  "optimal": {:hex, :optimal, "0.3.6", "46bbf52fbbbd238cda81e02560caa84f93a53c75620f1fe19e81e4ae7b07d1dd", [:mix], [], "hexpm"},
   "parallel_stream": {:hex, :parallel_stream, "1.0.6", "b967be2b23f0f6787fab7ed681b4c45a215a81481fb62b01a5b750fa8f30f76c", [:mix], []},
   "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm"},
   "phoenix": {:hex, :phoenix, "1.3.4", "aaa1b55e5523083a877bcbe9886d9ee180bf2c8754905323493c2ac325903dc5", [:mix], [{:cowboy, "~> 1.0", [hex: :cowboy, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.0", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.3.3 or ~> 1.4", [hex: :plug, repo: "hexpm", optional: false]}, {:poison, "~> 2.2 or ~> 3.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm"},
@@ -92,6 +95,10 @@
   "ranch": {:hex, :ranch, "1.3.2", "e4965a144dc9fbe70e5c077c65e73c57165416a901bd02ea899cfd95aa890986", [:rebar3], []},
   "set_locale": {:git, "https://github.com/minifast/set_locale.git", "da9ae029642bc0fbd9212c2aaf86c0adca70c084", [branch: "master"]},
   "sobelow": {:hex, :sobelow, "0.7.1", "01a52ea8a19be0aa41ce969746f057e90f3994f1607c771968359718bd0e6988", [:mix], [], "hexpm"},
+  "spandex": {:git, "https://github.com/spandex-project/spandex.git", "92992b4aaf3d92e031c2417ff2e6c9e94d27fe36", [branch: "allow-setting-trace-key"]},
+  "spandex_datadog": {:hex, :spandex_datadog, "0.3.1", "984d27ad1f45cfd243509692f0f63b900a23b79566c529a644c7f3a2b4120603", [:mix], [{:msgpax, "~> 1.1", [hex: :msgpax, repo: "hexpm", optional: false]}, {:spandex, "~> 2.3", [hex: :spandex, repo: "hexpm", optional: false]}], "hexpm"},
+  "spandex_ecto": {:hex, :spandex_ecto, "0.4.0", "deaeaddc11a35f1c551206c53d09bb93a0da5808dbef751430e465c8c7de01d3", [:mix], [{:spandex, "~> 2.2", [hex: :spandex, repo: "hexpm", optional: false]}], "hexpm"},
+  "spandex_phoenix": {:hex, :spandex_phoenix, "0.3.0", "48b0a426bbe4eea3e579bbea77d5eb5a8d4b83d33c95616f9ba64b3ce2faef6c", [:mix], [{:plug, "~> 1.3", [hex: :plug, repo: "hexpm", optional: false]}, {:spandex, "~> 2.2", [hex: :spandex, repo: "hexpm", optional: false]}], "hexpm"},
   "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"},
   "timex": {:hex, :timex, "3.4.1", "e63fc1a37453035e534c3febfe9b6b9e18583ec7b37fd9c390efdef97397d70b", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"},
   "timex_ecto": {:hex, :timex_ecto, "3.3.0", "d5bdef09928e7a60f10a0baa47ce653f29b43d6fee87b30b236b216d0e36b98d", [:mix], [{:ecto, "~> 2.2", [hex: :ecto, repo: "hexpm", optional: false]}, {:timex, "~> 3.1", [hex: :timex, repo: "hexpm", optional: false]}], "hexpm"},