Merge branch 'master' into wsa-async-load-blocks-validated

.credo.exs

@@ -75,8 +75,9 @@
         # Priority values are: `low, normal, high, higher`
         #
         {Credo.Check.Design.AliasUsage,
-         excluded_namespaces: ~w(Block Blocks Import Socket Task),
-         excluded_lastnames: ~w(Address DateTime Exporter Fetcher Full Instrumenter Monitor Name Number Repo Time Unit),
+         excluded_namespaces: ~w(Block Blocks Import Socket SpandexDatadog Task),
+         excluded_lastnames:
+           ~w(Address DateTime Exporter Fetcher Full Instrumenter Monitor Name Number Repo Spec Time Unit),
          priority: :low},
 
         # For some checks, you can also set other parameters

README.md

@@ -7,7 +7,7 @@
 <h1 align="center">BlockScout</h1>
 <p align="center">Blockchain Explorer for inspecting and analyzing EVM Chains.</p>
 <div align="center">
-  
+
 [![CircleCI](https://circleci.com/gh/poanetwork/blockscout.svg?style=svg&circle-token=f8823a3d0090407c11f87028c73015a331dbf604)](https://circleci.com/gh/poanetwork/blockscout) [![Coverage Status](https://coveralls.io/repos/github/poanetwork/blockscout/badge.svg?branch=master)](https://coveralls.io/github/poanetwork/blockscout?branch=master) [![Join the chat at https://gitter.im/poanetwork/blockscout](https://badges.gitter.im/poanetwork/blockscout.svg)](https://gitter.im/poanetwork/blockscout?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
 </div>
@@ -283,6 +283,33 @@ BlockScout is setup to export [Prometheus](https://prometheus.io/) metrics at `/
    3. Click "Load"
 6. View the dashboards.  (You will need to click-around and use BlockScout for the web-related metrics to show up.)
 
+## Tracing
+
+Blockscout supports tracing via
+[Spandex](http://git@github.com:spandex-project/spandex.git). Each application
+has its own tracer, that is configured internally to that application. In order
+to enable it, visit each application's `config/<env>.ex` and update its tracer
+configuration to change `disabled?: true` to `disabled?: false`. Do this for
+each application you'd like included in your trace data.
+
+Currently, only [Datadog](https://www.datadoghq.com/) is supported as a
+tracing backend, but more will be added soon.
+
+### DataDog
+
+If you would like to use DataDog, after enabling `Spandex`, set
+`"DATADOG_HOST"` and `"DATADOG_PORT"` environment variables to the
+host/port that your Datadog agent is running on. For more information on
+Datadog and the Datadog agent, see their
+[documentation](https://docs.datadoghq.com/).
+
+### Other
+
+If you want to use a different  backend, remove the
+`SpandexDatadog.ApiServer` `Supervisor.child_spec` from
+`Explorer.Application` and follow any instructions provided in `Spandex`
+for setting up that backend.
+
 ## Memory Usage
 
 The work queues for building the index of all blocks, balances (coin and token), and internal transactions can grow quite large.   By default, the soft-limit is 1 GiB, which can be changed in `apps/indexer/config/config.exs`:

apps/block_scout_web/config/config.exs

@@ -18,7 +18,7 @@ config :block_scout_web, BlockScoutWeb.Chain,
 
 # Configures the endpoint
 config :block_scout_web, BlockScoutWeb.Endpoint,
-  instrumenters: [BlockScoutWeb.Prometheus.Instrumenter],
+  instrumenters: [BlockScoutWeb.Prometheus.Instrumenter, SpandexPhoenix.Instrumenter],
   url: [
     host: "localhost",
     path: System.get_env("NETWORK_PATH") || "/"
@@ -26,6 +26,11 @@ config :block_scout_web, BlockScoutWeb.Endpoint,
   render_errors: [view: BlockScoutWeb.ErrorView, accepts: ~w(html json)],
   pubsub: [name: BlockScoutWeb.PubSub, adapter: Phoenix.PubSub.PG2]
 
+config :block_scout_web, BlockScoutWeb.Tracer,
+  service: :block_scout_web,
+  adapter: SpandexDatadog.Adapter,
+  trace_key: :blockscout
+
 # Configures gettext
 config :block_scout_web, BlockScoutWeb.Gettext, locales: ~w(en), default_locale: "en"
 
@@ -46,6 +51,8 @@ config :logger, :block_scout_web,
   metadata: [:application, :request_id],
   metadata_filter: [application: :block_scout_web]
 
+config :spandex_phoenix, tracer: BlockScoutWeb.Tracer
+
 config :wobserver,
   # return only the local node
   discovery: :none,

apps/block_scout_web/config/dev.exs

@@ -48,6 +48,8 @@ config :block_scout_web, BlockScoutWeb.Endpoint,
     ]
   ]
 
+config :block_scout_web, BlockScoutWeb.Tracer, env: "dev", disabled?: true
+
 config :logger, :block_scout_web,
   level: :debug,
   path: Path.absname("logs/dev/block_scout_web.log")

apps/block_scout_web/config/prod.exs

@@ -24,6 +24,8 @@ config :block_scout_web, BlockScoutWeb.Endpoint,
     port: System.get_env("PORT")
   ]
 
+config :block_scout_web, BlockScoutWeb.Tracer, env: "production", disabled?: true
+
 config :logger, :block_scout_web,
   level: :info,
   path: Path.absname("logs/prod/block_scout_web.log"),

apps/block_scout_web/config/test.exs

@@ -9,6 +9,8 @@ config :block_scout_web, BlockScoutWeb.Endpoint,
   secret_key_base: "27Swe6KtEtmN37WyEYRjKWyxYULNtrxlkCEKur4qoV+Lwtk8lafsR16ifz1XBBYj",
   server: true
 
+config :block_scout_web, BlockScoutWeb.Tracer, disabled?: false
+
 config :logger, :block_scout_web,
   level: :warn,
   path: Path.absname("logs/test/block_scout_web.log")

apps/block_scout_web/lib/block_scout_web/endpoint.ex

@@ -68,6 +68,8 @@ defmodule BlockScoutWeb.Endpoint do
     signing_salt: "iC2ksJHS"
   )
 
+  use SpandexPhoenix
+
   plug(BlockScoutWeb.Prometheus.Exporter)
 
   plug(BlockScoutWeb.Router)

apps/block_scout_web/lib/block_scout_web/etherscan.ex

@@ -1012,7 +1012,7 @@ defmodule BlockScoutWeb.Etherscan do
   @account_txlistinternal_action %{
     name: "txlistinternal",
     description:
-      "Get internal transactions by transaction or address hash. Up to a maximum of 10,000 internal transactions.",
+      "Get internal transactions by transaction or address hash. Up to a maximum of 10,000 internal transactions. Also available through a GraphQL 'transaction' query.",
     required_params: [
       %{
         key: "txhash",

apps/block_scout_web/lib/block_scout_web/resolvers/internal_transaction.ex

@@ -0,0 +1,23 @@
+defmodule BlockScoutWeb.Resolvers.InternalTransaction do
+  @moduledoc false
+
+  alias Absinthe.Relay.Connection
+  alias Explorer.Chain.Transaction
+  alias Explorer.{GraphQL, Repo}
+
+  def get_by(%{transaction_hash: _, index: _} = args) do
+    GraphQL.get_internal_transaction(args)
+  end
+
+  def get_by(%Transaction{} = transaction, args, _) do
+    transaction
+    |> GraphQL.transaction_to_internal_transactions_query()
+    |> Connection.from_query(&Repo.all/1, args, options(args))
+  end
+
+  defp options(%{before: _}), do: []
+
+  defp options(%{count: count}), do: [count: count]
+
+  defp options(_), do: []
+end

apps/block_scout_web/lib/block_scout_web/resolvers/transaction.ex

@@ -8,7 +8,7 @@ defmodule BlockScoutWeb.Resolvers.Transaction do
   def get_by(_, %{hash: hash}, _) do
     case Chain.hash_to_transaction(hash) do
       {:ok, transaction} -> {:ok, transaction}
-      {:error, :not_found} -> {:error, "Transaction hash #{hash} was not found."}
+      {:error, :not_found} -> {:error, "Transaction not found."}
     end
   end
 

apps/block_scout_web/lib/block_scout_web/schema.ex

@@ -6,8 +6,16 @@ defmodule BlockScoutWeb.Schema do
 
   alias Absinthe.Middleware.Dataloader, as: AbsintheMiddlewareDataloader
   alias Absinthe.Plugin, as: AbsinthePlugin
-  alias BlockScoutWeb.Resolvers.{Address, Block, Transaction}
+
+  alias BlockScoutWeb.Resolvers.{
+    Address,
+    Block,
+    InternalTransaction,
+    Transaction
+  }
+
   alias Explorer.Chain
+  alias Explorer.Chain.InternalTransaction, as: ExplorerChainInternalTransaction
   alias Explorer.Chain.Transaction, as: ExplorerChainTransaction
 
   import_types(BlockScoutWeb.Schema.Types)
@@ -17,6 +25,9 @@ defmodule BlockScoutWeb.Schema do
       %ExplorerChainTransaction{}, _ ->
         :transaction
 
+      %ExplorerChainInternalTransaction{}, _ ->
+        :internal_transaction
+
       _, _ ->
         nil
     end)
@@ -29,6 +40,11 @@ defmodule BlockScoutWeb.Schema do
           {:ok, hash} = Chain.string_to_transaction_hash(transaction_hash_string)
           Transaction.get_by(%{}, %{hash: hash}, %{})
 
+        %{type: :internal_transaction, id: id}, _ ->
+          %{"transaction_hash" => transaction_hash_string, "index" => index} = Jason.decode!(id)
+          {:ok, transaction_hash} = Chain.string_to_transaction_hash(transaction_hash_string)
+          InternalTransaction.get_by(%{transaction_hash: transaction_hash, index: index})
+
         _, _ ->
           {:error, "Unknown node"}
       end)

apps/block_scout_web/lib/block_scout_web/schema/scalars.ex

@@ -99,4 +99,18 @@ defmodule BlockScoutWeb.Schema.Scalars do
     value(:ok)
     value(:error)
   end
+
+  enum :call_type do
+    value(:call)
+    value(:callcode)
+    value(:delegatecall)
+    value(:staticcall)
+  end
+
+  enum :type do
+    value(:call)
+    value(:create)
+    value(:reward)
+    value(:selfdestruct)
+  end
 end

apps/block_scout_web/lib/block_scout_web/schema/types.ex

@@ -6,12 +6,16 @@ defmodule BlockScoutWeb.Schema.Types do
 
   import Absinthe.Resolution.Helpers
 
-  alias BlockScoutWeb.Resolvers.Transaction
+  alias BlockScoutWeb.Resolvers.{
+    InternalTransaction,
+    Transaction
+  }
 
   import_types(Absinthe.Type.Custom)
   import_types(BlockScoutWeb.Schema.Scalars)
 
   connection(node_type: :transaction)
+  connection(node_type: :internal_transaction)
 
   @desc """
   A stored representation of a Web3 address.
@@ -60,6 +64,30 @@ defmodule BlockScoutWeb.Schema.Types do
     field(:parent_hash, :full_hash)
   end
 
+  @desc """
+  Models internal transactions.
+  """
+  node object(:internal_transaction, id_fetcher: &internal_transaction_id_fetcher/2) do
+    field(:call_type, :call_type)
+    field(:created_contract_code, :data)
+    field(:error, :string)
+    field(:gas, :decimal)
+    field(:gas_used, :decimal)
+    field(:index, :integer)
+    field(:init, :data)
+    field(:input, :data)
+    field(:output, :data)
+    field(:trace_address, :json)
+    field(:type, :type)
+    field(:value, :wei)
+    field(:block_number, :integer)
+    field(:transaction_index, :integer)
+    field(:created_contract_address_hash, :address_hash)
+    field(:from_address_hash, :address_hash)
+    field(:to_address_hash, :address_hash)
+    field(:transaction_hash, :full_hash)
+  end
+
   @desc """
   The representation of a verified Smart Contract.
 
@@ -99,6 +127,19 @@ defmodule BlockScoutWeb.Schema.Types do
     field(:from_address_hash, :address_hash)
     field(:to_address_hash, :address_hash)
     field(:created_contract_address_hash, :address_hash)
+
+    connection field(:internal_transactions, node_type: :internal_transaction) do
+      arg(:count, :integer)
+      resolve(&InternalTransaction.get_by/3)
+
+      complexity(fn
+        %{first: first}, child_complexity ->
+          first * child_complexity
+
+        %{last: last}, child_complexity ->
+          last * child_complexity
+      end)
+    end
   end
 
   @desc """
@@ -113,4 +154,8 @@ defmodule BlockScoutWeb.Schema.Types do
   end
 
   def transaction_id_fetcher(%{hash: hash}, _), do: to_string(hash)
+
+  def internal_transaction_id_fetcher(%{transaction_hash: transaction_hash, index: index}, _) do
+    Jason.encode!(%{transaction_hash: to_string(transaction_hash), index: index})
+  end
 end

apps/block_scout_web/lib/block_scout_web/tracer.ex

@@ -0,0 +1,5 @@
+defmodule BlockScoutWeb.Tracer do
+  @moduledoc false
+
+  use Spandex.Tracer, otp_app: :block_scout_web
+end

apps/block_scout_web/mix.exs

@@ -112,6 +112,12 @@ defmodule BlockScoutWeb.Mixfile do
       {:prometheus_process_collector, "~> 1.3"},
       {:qrcode, "~> 0.1.0"},
       {:sobelow, ">= 0.7.0", only: [:dev, :test], runtime: false},
+      # Tracing
+      {:spandex, github: "spandex-project/spandex", branch: "allow-setting-trace-key", override: true},
+      # `:spandex` integration with Datadog
+      {:spandex_datadog, "~> 0.3.1"},
+      # `:spandex` tracing of `:phoenix`
+      {:spandex_phoenix, "~> 0.3.0"},
       {:timex, "~> 3.4"},
       {:wallaby, "~> 0.20", only: [:test], runtime: false},
       {:wobserver, "~> 0.1.8"}

apps/block_scout_web/test/block_scout_web/controllers/tokens/inventory_controller_test.ex

@@ -0,0 +1,121 @@
+defmodule BlockScoutWeb.Tokens.InventoryControllerTest do
+  use BlockScoutWeb.ConnCase
+
+  describe "GET index/3" do
+    test "with invalid address hash", %{conn: conn} do
+      conn = get(conn, token_inventory_path(conn, :index, "invalid_address"))
+
+      assert html_response(conn, 404)
+    end
+
+    test "with a token that doesn't exist", %{conn: conn} do
+      address = build(:address)
+      conn = get(conn, token_inventory_path(conn, :index, address.hash))
+
+      assert html_response(conn, 404)
+    end
+
+    test "successfully renders the page", %{conn: conn} do
+      token_contract_address = insert(:contract_address)
+      token = insert(:token, type: "ERC-721", contract_address: token_contract_address)
+
+      transaction =
+        :transaction
+        |> insert()
+        |> with_block()
+
+      insert(
+        :token_transfer,
+        transaction: transaction,
+        token_contract_address: token_contract_address,
+        token: token
+      )
+
+      conn =
+        get(
+          conn,
+          token_inventory_path(conn, :index, token_contract_address.hash)
+        )
+
+      assert html_response(conn, 200)
+    end
+
+    test "returns next page of results based on last seen token balance", %{conn: conn} do
+      token = insert(:token, type: "ERC-721")
+
+      transaction =
+        :transaction
+        |> insert()
+        |> with_block()
+
+      second_page_token_balances =
+        Enum.map(
+          1..50,
+          &insert(
+            :token_transfer,
+            transaction: transaction,
+            token_contract_address: token.contract_address,
+            token: token,
+            token_id: &1 + 1000
+          )
+        )
+
+      conn =
+        get(conn, token_inventory_path(conn, :index, token.contract_address_hash), %{
+          "token_id" => "999"
+        })
+
+      assert Enum.map(conn.assigns.unique_tokens, & &1.token_id) == Enum.map(second_page_token_balances, & &1.token_id)
+    end
+
+    test "next_page_params exists if not on last page", %{conn: conn} do
+      token = insert(:token, type: "ERC-721")
+
+      transaction =
+        :transaction
+        |> insert()
+        |> with_block()
+
+      Enum.each(
+        1..51,
+        &insert(
+          :token_transfer,
+          transaction: transaction,
+          token_contract_address: token.contract_address,
+          token: token,
+          token_id: &1 + 1000
+        )
+      )
+
+      expected_next_page_params = %{
+        "token_id" => to_string(token.contract_address_hash),
+        "unique_token" => 1050
+      }
+
+      conn = get(conn, token_inventory_path(conn, :index, token.contract_address_hash))
+
+      assert conn.assigns.next_page_params == expected_next_page_params
+    end
+
+    test "next_page_params are empty if on last page", %{conn: conn} do
+      token = insert(:token, type: "ERC-721")
+
+      transaction =
+        :transaction
+        |> insert()
+        |> with_block()
+
+      insert(
+        :token_transfer,
+        transaction: transaction,
+        token_contract_address: token.contract_address,
+        token: token,
+        token_id: 1000
+      )
+
+      conn = get(conn, token_inventory_path(conn, :index, token.contract_address_hash))
+
+      refute conn.assigns.next_page_params
+    end
+  end
+end

apps/block_scout_web/test/block_scout_web/schema/query/address_test.exs

@@ -515,33 +515,13 @@ defmodule BlockScoutWeb.Schema.Query.AddressTest do
         |> List.last()
         |> Map.get("cursor")
 
-      query3 = """
-      query ($hash: AddressHash!, $first: Int!, $after: Int!) {
-        address(hash: $hash) {
-          transactions(first: $first, after: $after) {
-            page_info {
-              has_next_page
-              has_previous_page
-            }
-            edges {
-              node {
-                hash
-                block_number
-              }
-              cursor
-            }
-          }
-        }
-      }
-      """
-
       variables3 = %{
         "hash" => to_string(address.hash),
         "first" => 3,
         "after" => last_cursor_page2
       }
 
-      conn = get(conn, "/graphql", query: query3, variables: variables3)
+      conn = get(conn, "/graphql", query: query2, variables: variables3)
 
       %{"data" => %{"address" => %{"transactions" => page3}}} = json_response(conn, 200)
 

apps/block_scout_web/test/block_scout_web/schema/query/node_test.exs

@@ -31,5 +31,100 @@ defmodule BlockScoutWeb.Schema.Query.NodeTest do
                }
              }
     end
+
+    test "with 'id' for non-existent transaction", %{conn: conn} do
+      transaction = build(:transaction)
+
+      query = """
+      query($id: ID!) {
+        node(id: $id) {
+          ... on Transaction {
+            id
+            hash
+          }
+        }
+      }
+      """
+
+      id = Base.encode64("Transaction:#{transaction.hash}")
+
+      variables = %{"id" => id}
+
+      conn = get(conn, "/graphql", query: query, variables: variables)
+
+      %{"errors" => [error]} = json_response(conn, 200)
+
+      assert error["message"] == "Transaction not found."
+    end
+
+    test "with valid argument 'id' for an internal transaction", %{conn: conn} do
+      transaction = insert(:transaction)
+
+      internal_transaction = insert(:internal_transaction, transaction: transaction, index: 0)
+
+      query = """
+      query($id: ID!) {
+        node(id: $id) {
+          ... on InternalTransaction {
+            id
+            transaction_hash
+            index
+          }
+        }
+      }
+      """
+
+      id =
+        %{transaction_hash: to_string(transaction.hash), index: internal_transaction.index}
+        |> Jason.encode!()
+        |> (fn unique_id -> "InternalTransaction:#{unique_id}" end).()
+        |> Base.encode64()
+
+      variables = %{"id" => id}
+
+      conn = get(conn, "/graphql", query: query, variables: variables)
+
+      assert json_response(conn, 200) == %{
+               "data" => %{
+                 "node" => %{
+                   "id" => id,
+                   "transaction_hash" => to_string(transaction.hash),
+                   "index" => internal_transaction.index
+                 }
+               }
+             }
+    end
+
+    test "with 'id' for non-existent internal transaction", %{conn: conn} do
+      transaction = build(:transaction)
+
+      internal_transaction = build(:internal_transaction, transaction: transaction, index: 0)
+
+      query = """
+      query($id: ID!) {
+        node(id: $id) {
+          ... on InternalTransaction {
+            id
+            transaction_hash
+            index
+          }
+        }
+      }
+      """
+
+      id =
+        %{transaction_hash: to_string(transaction.hash), index: internal_transaction.index}
+        |> Jason.encode!()
+        |> (fn unique_id -> "InternalTransaction:#{unique_id}" end).()
+        |> Base.encode64()
+
+      variables = %{"id" => id}
+
+      conn = get(conn, "/graphql", query: query, variables: variables)
+
+      %{"errors" => [error]} = json_response(conn, 200)
+
+      assert error["message"] == "Internal transaction not found."
+    end
   end
 end

apps/block_scout_web/test/block_scout_web/schema/query/transaction_test.exs

@@ -81,7 +81,7 @@ defmodule BlockScoutWeb.Schema.Query.TransactionTest do
       conn = get(conn, "/graphql", query: query, variables: variables)
 
       assert %{"errors" => [error]} = json_response(conn, 200)
-      assert error["message"] =~ ~s(Transaction hash #{transaction.hash} was not found)
+      assert error["message"] == "Transaction not found."
     end
 
     test "errors if argument 'hash' is missing", %{conn: conn} do
@@ -116,4 +116,396 @@ defmodule BlockScoutWeb.Schema.Query.TransactionTest do
       assert error["message"] =~ ~s(Argument "hash" has invalid value)
     end
   end
+
+  describe "transaction internal_transactions field" do
+    test "returns all expected internal_transaction fields", %{conn: conn} do
+      address = insert(:address)
+      contract_address = insert(:contract_address)
+
+      block = insert(:block)
+
+      transaction =
+        :transaction
+        |> insert(from_address: address)
+        |> with_contract_creation(contract_address)
+        |> with_block(block)
+
+      internal_transaction_attributes = %{
+        transaction: transaction,
+        index: 0,
+        from_address: address,
+        call_type: :call
+      }
+
+      internal_transaction =
+        :internal_transaction_create
+        |> insert(internal_transaction_attributes)
+        |> with_contract_creation(contract_address)
+
+      query = """
+      query ($hash: FullHash!, $first: Int!) {
+        transaction(hash: $hash) {
+          internal_transactions(first: $first) {
+            edges {
+              node {
+                call_type
+                created_contract_code
+                error
+                gas
+                gas_used
+                index
+                init
+                input
+                output
+                trace_address
+                type
+                value
+                block_number
+                transaction_index
+                created_contract_address_hash
+                from_address_hash
+                to_address_hash
+                transaction_hash
+              }
+            }
+          }
+        }
+      }
+      """
+
+      variables = %{
+        "hash" => to_string(transaction.hash),
+        "first" => 1
+      }
+
+      conn = get(conn, "/graphql", query: query, variables: variables)
+
+      assert json_response(conn, 200) == %{
+               "data" => %{
+                 "transaction" => %{
+                   "internal_transactions" => %{
+                     "edges" => [
+                       %{
+                         "node" => %{
+                           "call_type" => internal_transaction.call_type |> to_string() |> String.upcase(),
+                           "created_contract_code" => to_string(internal_transaction.created_contract_code),
+                           "error" => internal_transaction.error,
+                           "gas" => to_string(internal_transaction.gas),
+                           "gas_used" => to_string(internal_transaction.gas_used),
+                           "index" => internal_transaction.index,
+                           "init" => to_string(internal_transaction.init),
+                           "input" => nil,
+                           "output" => nil,
+                           "trace_address" => Jason.encode!(internal_transaction.trace_address),
+                           "type" => internal_transaction.type |> to_string() |> String.upcase(),
+                           "value" => to_string(internal_transaction.value.value),
+                           "block_number" => internal_transaction.block_number,
+                           "transaction_index" => internal_transaction.transaction_index,
+                           "created_contract_address_hash" =>
+                             to_string(internal_transaction.created_contract_address_hash),
+                           "from_address_hash" => to_string(internal_transaction.from_address_hash),
+                           "to_address_hash" => nil,
+                           "transaction_hash" => to_string(internal_transaction.transaction_hash)
+                         }
+                       }
+                     ]
+                   }
+                 }
+               }
+             }
+    end
+
+    test "with transaction with zero internal transactions", %{conn: conn} do
+      address = insert(:address)
+
+      block = insert(:block)
+
+      transaction =
+        :transaction
+        |> insert(from_address: address)
+        |> with_block(block)
+
+      query = """
+      query ($hash: FullHash!, $first: Int!) {
+        transaction(hash: $hash) {
+          internal_transactions(first: $first) {
+            edges {
+              node {
+                index
+                transaction_hash
+              }
+            }
+          }
+        }
+      }
+      """
+
+      variables = %{
+        "hash" => to_string(transaction.hash),
+        "first" => 1
+      }
+
+      conn = get(conn, "/graphql", query: query, variables: variables)
+
+      assert json_response(conn, 200) == %{
+               "data" => %{
+                 "transaction" => %{
+                   "internal_transactions" => %{
+                     "edges" => []
+                   }
+                 }
+               }
+             }
+    end
+
+    test "internal transactions are ordered by ascending index", %{conn: conn} do
+      transaction = insert(:transaction)
+
+      insert(:internal_transaction, transaction: transaction, index: 2)
+      insert(:internal_transaction, transaction: transaction, index: 0)
+      insert(:internal_transaction, transaction: transaction, index: 1)
+
+      query = """
+      query ($hash: FullHash!, $first: Int!) {
+        transaction(hash: $hash) {
+          internal_transactions(first: $first) {
+            edges {
+              node {
+                index
+                transaction_hash
+              }
+            }
+          }
+        }
+      }
+      """
+
+      variables = %{
+        "hash" => to_string(transaction.hash),
+        "first" => 3
+      }
+
+      response =
+        conn
+        |> get("/graphql", query: query, variables: variables)
+        |> json_response(200)
+
+      internal_transactions = get_in(response, ["data", "transaction", "internal_transactions", "edges"])
+
+      index_order = Enum.map(internal_transactions, & &1["node"]["index"])
+
+      assert index_order == Enum.sort(index_order)
+    end
+
+    test "complexity correlates to first or last argument", %{conn: conn} do
+      transaction = insert(:transaction)
+
+      query1 = """
+      query ($hash: FullHash!, $first: Int!) {
+        transaction(hash: $hash) {
+          internal_transactions(first: $first) {
+            edges {
+              node {
+                index
+                transaction_hash
+              }
+            }
+          }
+        }
+      }
+      """
+
+      variables1 = %{
+        "hash" => to_string(transaction.hash),
+        "first" => 55
+      }
+
+      response1 =
+        conn
+        |> get("/graphql", query: query1, variables: variables1)
+        |> json_response(200)
+
+      assert %{"errors" => [error1, error2, error3]} = response1
+      assert error1["message"] =~ ~s(Field internal_transactions is too complex)
+      assert error2["message"] =~ ~s(Field transaction is too complex)
+      assert error3["message"] =~ ~s(Operation is too complex)
+
+      query2 = """
+      query ($hash: FullHash!, $last: Int!, $count: Int!) {
+        transaction(hash: $hash) {
+          internal_transactions(last: $last, count: $count) {
+            edges {
+              node {
+                index
+                transaction_hash
+              }
+            }
+          }
+        }
+      }
+      """
+
+      variables2 = %{
+        "hash" => to_string(transaction.hash),
+        "last" => 55,
+        "count" => 100
+      }
+
+      response2 =
+        conn
+        |> get("/graphql", query: query2, variables: variables2)
+        |> json_response(200)
+
+      assert %{"errors" => [error1, error2, error3]} = response2
+      assert error1["message"] =~ ~s(Field internal_transactions is too complex)
+      assert error2["message"] =~ ~s(Field transaction is too complex)
+      assert error3["message"] =~ ~s(Operation is too complex)
+    end
+
+    test "with 'last' and 'count' arguments", %{conn: conn} do
+      # "`last: N` must always be acompanied by either a `before:` argument to
+      # the query, or an explicit `count:` option to the `from_query` call.
+      # Otherwise it is impossible to derive the required offset."
+      # https://hexdocs.pm/absinthe_relay/Absinthe.Relay.Connection.html#from_query/4
+      #
+      # This test ensures support for a 'count' argument.
+
+      transaction = insert(:transaction)
+
+      insert(:internal_transaction, transaction: transaction, index: 2)
+      insert(:internal_transaction, transaction: transaction, index: 0)
+      insert(:internal_transaction, transaction: transaction, index: 1)
+
+      query = """
+      query ($hash: FullHash!, $last: Int!, $count: Int!) {
+        transaction(hash: $hash) {
+          internal_transactions(last: $last, count: $count) {
+            edges {
+              node {
+                index
+                transaction_hash
+              }
+            }
+          }
+        }
+      }
+      """
+
+      variables = %{
+        "hash" => to_string(transaction.hash),
+        "last" => 1,
+        "count" => 3
+      }
+
+      [internal_transaction] =
+        conn
+        |> get("/graphql", query: query, variables: variables)
+        |> json_response(200)
+        |> get_in(["data", "transaction", "internal_transactions", "edges"])
+
+      assert internal_transaction["node"]["index"] == 2
+    end
+
+    test "pagination support with 'first' and 'after' arguments", %{conn: conn} do
+      transaction = insert(:transaction)
+
+      for index <- 0..5 do
+        insert(:internal_transaction_create, transaction: transaction, index: index)
+      end
+
+      query1 = """
+      query ($hash: AddressHash!, $first: Int!) {
+        transaction(hash: $hash) {
+          internal_transactions(first: $first) {
+            page_info {
+              has_next_page
+              has_previous_page
+            }
+            edges {
+              node {
+                index
+                transaction_hash
+              }
+              cursor
+            }
+          }
+        }
+      }
+      """
+
+      variables1 = %{
+        "hash" => to_string(transaction.hash),
+        "first" => 2
+      }
+
+      conn = get(conn, "/graphql", query: query1, variables: variables1)
+
+      %{"data" => %{"transaction" => %{"internal_transactions" => page1}}} = json_response(conn, 200)
+
+      assert page1["page_info"] == %{"has_next_page" => true, "has_previous_page" => false}
+      assert Enum.all?(page1["edges"], &(&1["node"]["index"] in 0..1))
+
+      last_cursor_page1 =
+        page1
+        |> Map.get("edges")
+        |> List.last()
+        |> Map.get("cursor")
+
+      query2 = """
+      query ($hash: AddressHash!, $first: Int!, $after: ID!) {
+        transaction(hash: $hash) {
+          internal_transactions(first: $first, after: $after) {
+            page_info {
+              has_next_page
+              has_previous_page
+            }
+            edges {
+              node {
+                index
+                transaction_hash
+              }
+              cursor
+            }
+          }
+        }
+      }
+      """
+
+      variables2 = %{
+        "hash" => to_string(transaction.hash),
+        "first" => 2,
+        "after" => last_cursor_page1
+      }
+
+      page2 =
+        conn
+        |> get("/graphql", query: query2, variables: variables2)
+        |> json_response(200)
+        |> get_in(["data", "transaction", "internal_transactions"])
+
+      assert page2["page_info"] == %{"has_next_page" => true, "has_previous_page" => true}
+      assert Enum.all?(page2["edges"], &(&1["node"]["index"] in 2..3))
+
+      last_cursor_page2 =
+        page2
+        |> Map.get("edges")
+        |> List.last()
+        |> Map.get("cursor")
+
+      variables3 = %{
+        "hash" => to_string(transaction.hash),
+        "first" => 2,
+        "after" => last_cursor_page2
+      }
+
+      page3 =
+        conn
+        |> get("/graphql", query: query2, variables: variables3)
+        |> json_response(200)
+        |> get_in(["data", "transaction", "internal_transactions"])
+
+      assert page3["page_info"] == %{"has_next_page" => false, "has_previous_page" => true}
+      assert Enum.all?(page3["edges"], &(&1["node"]["index"] in 4..5))
+    end
+  end
 end

apps/ethereum_jsonrpc/config/config.exs

@@ -1,11 +1,5 @@
 use Mix.Config
 
-config :logger, :ethereum_jsonrpc,
-  # keep synced with `config/config.exs`
-  format: "$time $metadata[$level] $message\n",
-  metadata: [:application, :request_id],
-  metadata_filter: [application: :ethereum_jsonrpc]
-
 config :ethereum_jsonrpc, EthereumJSONRPC.RequestCoordinator,
   rolling_window_opts: [
     window_count: 12,
@@ -15,6 +9,17 @@ config :ethereum_jsonrpc, EthereumJSONRPC.RequestCoordinator,
   wait_per_timeout: :timer.seconds(20),
   max_jitter: :timer.seconds(2)
 
+config :ethereum_jsonrpc, EthereumJSONRPC.Tracer,
+  service: :ethereum_jsonrpc,
+  adapter: SpandexDatadog.Adapter,
+  trace_key: :blockscout
+
+config :logger, :ethereum_jsonrpc,
+  # keep synced with `config/config.exs`
+  format: "$time $metadata[$level] $message\n",
+  metadata: [:application, :request_id],
+  metadata_filter: [application: :ethereum_jsonrpc]
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"

apps/ethereum_jsonrpc/config/dev.exs

@@ -1,5 +1,7 @@
 use Mix.Config
 
+config :ethereum_jsonrpc, EthereumJSONRPC.Tracer, env: "dev", disabled?: true
+
 config :logger, :ethereum_jsonrpc,
   level: :debug,
   path: Path.absname("logs/dev/ethereum_jsonrpc.log")

apps/ethereum_jsonrpc/config/prod.exs

@@ -1,5 +1,7 @@
 use Mix.Config
 
+config :ethereum_jsonrpc, EthereumJSONRPC.Tracer, env: "production", disabled?: true
+
 config :logger, :ethereum_jsonrpc,
   level: :info,
   path: Path.absname("logs/prod/ethereum_jsonrpc.log"),

apps/ethereum_jsonrpc/config/test.exs

@@ -1,9 +1,5 @@
 use Mix.Config
 
-config :logger, :ethereum_jsonrpc,
-  level: :warn,
-  path: Path.absname("logs/test/ethereum_jsonrpc.log")
-
 config :ethereum_jsonrpc, EthereumJSONRPC.RequestCoordinator,
   rolling_window_opts: [
     window_count: 3,
@@ -12,3 +8,9 @@ config :ethereum_jsonrpc, EthereumJSONRPC.RequestCoordinator,
   ],
   wait_per_timeout: 2,
   max_jitter: 1
+
+config :ethereum_jsonrpc, EthereumJSONRPC.Tracer, disabled?: false
+
+config :logger, :ethereum_jsonrpc,
+  level: :warn,
+  path: Path.absname("logs/test/ethereum_jsonrpc.log")

apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/ganache.ex

@@ -0,0 +1,31 @@
+defmodule EthereumJSONRPC.Ganache do
+  @moduledoc """
+  Ethereum JSONRPC methods that are only supported by [Ganache](https://github.com/trufflesuite/ganache-core#implemented-methods).
+  """
+
+  @behaviour EthereumJSONRPC.Variant
+
+  @doc """
+  Block reward contract beneficiary fetching is not supported currently for Ganache.
+
+  To signal to the caller that fetching is not supported, `:ignore` is returned.
+  """
+  @impl EthereumJSONRPC.Variant
+  def fetch_beneficiaries(_block_range, _json_rpc_named_arguments), do: :ignore
+
+  @doc """
+  Internal transaction fetching is not currently supported for Ganache.
+
+  To signal to the caller that fetching is not supported, `:ignore` is returned.
+  """
+  @impl EthereumJSONRPC.Variant
+  def fetch_internal_transactions(_transactions_params, _json_rpc_named_arguments), do: :ignore
+
+  @doc """
+  Pending transaction fetching is not supported currently for Ganache.
+
+  To signal to the caller that fetching is not supported, `:ignore` is returned.
+  """
+  @impl EthereumJSONRPC.Variant
+  def fetch_pending_transactions(_json_rpc_named_arguments), do: :ignore
+end

apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/geth.ex

@@ -18,9 +18,7 @@ defmodule EthereumJSONRPC.Geth do
   def fetch_beneficiaries(_block_range, _json_rpc_named_arguments), do: :ignore
 
   @doc """
-  Internal transaction fetching is not supported currently for Geth.
-
-  To signal to the caller that fetching is not supported, `:ignore` is returned.
+  Fetches the `t:Explorer.Chain.InternalTransaction.changeset/2` params.
   """
   @impl EthereumJSONRPC.Variant
   def fetch_internal_transactions(transactions_params, json_rpc_named_arguments) when is_list(transactions_params) do

apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/request_coordinator.ex

@@ -42,7 +42,9 @@ defmodule EthereumJSONRPC.RequestCoordinator do
   With this configuration, timeouts are tracked for 6 windows of 10 seconds for a total of 1 minute.
   """
 
-  alias EthereumJSONRPC.{RollingWindow, Transport}
+  require EthereumJSONRPC.Tracer
+
+  alias EthereumJSONRPC.{RollingWindow, Tracer, Transport}
 
   @error_key :throttleable_error_count
 
@@ -63,9 +65,11 @@ defmodule EthereumJSONRPC.RequestCoordinator do
     if sleep_time <= throttle_timeout do
       :timer.sleep(sleep_time)
 
-      request
-      |> transport.json_rpc(transport_options)
-      |> handle_transport_response()
+      trace_request(request, fn ->
+        request
+        |> transport.json_rpc(transport_options)
+        |> handle_transport_response()
+      end)
     else
       :timer.sleep(throttle_timeout)
 
@@ -73,6 +77,18 @@ defmodule EthereumJSONRPC.RequestCoordinator do
     end
   end
 
+  defp trace_request([request | _], fun) do
+    trace_request(request, fun)
+  end
+
+  defp trace_request(%{method: method}, fun) do
+    Tracer.span "RequestCoordinator.perform/4", resource: method, service: :ethereum_jsonrpc do
+      fun.()
+    end
+  end
+
+  defp trace_request(_, fun), do: fun.()
+
   defp handle_transport_response({:error, {:bad_gateway, _}} = error) do
     RollingWindow.inc(table(), @error_key)
     error

apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/tracer.ex

@@ -0,0 +1,5 @@
+defmodule EthereumJSONRPC.Tracer do
+  @moduledoc false
+
+  use Spandex.Tracer, otp_app: :ethereum_jsonrpc
+end

apps/ethereum_jsonrpc/mix.exs

@@ -76,6 +76,10 @@ defmodule EthereumJsonrpc.MixProject do
       {:logger_file_backend, "~> 0.0.10"},
       # Mocking `EthereumJSONRPC.Transport` and `EthereumJSONRPC.HTTP` so we avoid hitting real chains for local testing
       {:mox, "~> 0.4", only: [:test]},
+      # Tracing
+      {:spandex, github: "spandex-project/spandex", branch: "allow-setting-trace-key", override: true},
+      # `:spandex` integration with Datadog
+      {:spandex_datadog, "~> 0.3.1"},
       # Convert unix timestamps in JSONRPC to DateTimes
       {:timex, "~> 3.4"},
       # Encode/decode function names and arguments

apps/explorer/config/config.exs

@@ -25,6 +25,11 @@ config :explorer, Explorer.Repo,
   loggers: [Explorer.Repo.PrometheusLogger, Ecto.LogEntry],
   migration_timestamps: [type: :utc_datetime]
 
+config :explorer, Explorer.Tracer,
+  service: :explorer,
+  adapter: SpandexDatadog.Adapter,
+  trace_key: :blockscout
+
 config :explorer, Explorer.Counters.TokenTransferCounter, enabled: true
 
 config :explorer, Explorer.Counters.TokenHoldersCounter, enabled: true, enable_consolidation: true
@@ -46,6 +51,11 @@ config :logger, :explorer,
   metadata: [:application, :request_id],
   metadata_filter: [application: :explorer]
 
+config :spandex_ecto, SpandexEcto.EctoLogger,
+  service: :ecto,
+  tracer: Explorer.Tracer,
+  otp_app: :explorer
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"

apps/explorer/config/dev.exs

@@ -9,6 +9,8 @@ config :explorer, Explorer.Repo,
   pool_timeout: 60_000,
   timeout: 80_000
 
+config :explorer, Explorer.Tracer, env: "dev", disabled?: true
+
 config :logger, :explorer,
   level: :debug,
   path: Path.absname("logs/dev/explorer.log")

apps/explorer/config/dev/ganache.exs

@@ -8,7 +8,7 @@ config :explorer,
       url: System.get_env("ETHEREUM_JSONRPC_HTTP_URL") || "http://localhost:7545",
       http_options: [recv_timeout: 60_000, timeout: 60_000, hackney: [pool: :ethereum_jsonrpc]]
     ],
-    variant: EthereumJSONRPC.Geth
+    variant: EthereumJSONRPC.Ganache
   ],
   subscribe_named_arguments: [
     transport: EthereumJSONRPC.WebSocket,
@@ -16,5 +16,5 @@ config :explorer,
       web_socket: EthereumJSONRPC.WebSocket.WebSocketClient,
       url: System.get_env("ETHEREUM_JSONRPC_WS_URL") || "ws://localhost:7545"
     ],
-    variant: EthereumJSONRPC.Geth
+    variant: EthereumJSONRPC.Ganache
   ]

apps/explorer/config/prod.exs

@@ -9,6 +9,8 @@ config :explorer, Explorer.Repo,
   prepare: :unnamed,
   timeout: 60_000
 
+config :explorer, Explorer.Tracer, env: "production", disabled?: true
+
 config :logger, :explorer,
   level: :info,
   path: Path.absname("logs/prod/explorer.log"),

apps/explorer/config/prod/ganache.exs

@@ -8,7 +8,7 @@ config :explorer,
       url: System.get_env("ETHEREUM_JSONRPC_HTTP_URL") || "http://localhost:7545",
       http_options: [recv_timeout: 60_000, timeout: 60_000, hackney: [pool: :ethereum_jsonrpc]]
     ],
-    variant: EthereumJSONRPC.Geth
+    variant: EthereumJSONRPC.Ganache
   ],
   subscribe_named_arguments: [
     transport: EthereumJSONRPC.WebSocket,
@@ -16,5 +16,5 @@ config :explorer,
       web_socket: EthereumJSONRPC.WebSocket.WebSocketClient,
       url: System.get_env("ETHEREUM_JSONRPC_WS_URL") || "ws://localhost:7545"
     ],
-    variant: EthereumJSONRPC.Geth
+    variant: EthereumJSONRPC.Ganache
   ]

apps/explorer/config/test.exs

@@ -3,6 +3,8 @@ use Mix.Config
 # Lower hashing rounds for faster tests
 config :bcrypt_elixir, log_rounds: 4
 
+config :explorer, Explorer.Counters.TokenHoldersCounter, enabled: true, enable_consolidation: false
+
 # Configure your database
 config :explorer, Explorer.Repo,
   adapter: Ecto.Adapters.Postgres,
@@ -17,7 +19,7 @@ config :explorer, Explorer.ExchangeRates, enabled: false
 
 config :explorer, Explorer.Market.History.Cataloger, enabled: false
 
-config :explorer, Explorer.Counters.TokenHoldersCounter, enabled: true, enable_consolidation: false
+config :explorer, Explorer.Tracer, disabled?: false
 
 config :logger, :explorer,
   level: :warn,

apps/explorer/config/test/ganache.exs

@@ -4,10 +4,10 @@ config :explorer,
   json_rpc_named_arguments: [
     transport: EthereumJSONRPC.Mox,
     transport_options: [],
-    variant: EthereumJSONRPC.Geth
+    variant: EthereumJSONRPC.Ganache
   ],
   subscribe_named_arguments: [
     transport: EthereumJSONRPC.Mox,
     transport_options: [],
-    variant: EthereumJSONRPC.Geth
+    variant: EthereumJSONRPC.Ganache
   ]

apps/explorer/lib/explorer/application.ex

@@ -15,6 +15,7 @@ defmodule Explorer.Application do
     # Children to start in all environments
     base_children = [
       Explorer.Repo,
+      Supervisor.Spec.worker(SpandexDatadog.ApiServer, [datadog_opts()]),
       Supervisor.child_spec({Task.Supervisor, name: Explorer.MarketTaskSupervisor}, id: Explorer.MarketTaskSupervisor),
       Supervisor.child_spec({Task.Supervisor, name: Explorer.TaskSupervisor}, id: Explorer.TaskSupervisor),
       {Registry, keys: :duplicate, name: Registry.ChainEvents, id: Registry.ChainEvents},
@@ -52,4 +53,14 @@ defmodule Explorer.Application do
       []
     end
   end
+
+  defp datadog_opts do
+    [
+      host: System.get_env("DATADOG_HOST") || "localhost",
+      port: System.get_env("DATADOG_PORT") || 8126,
+      batch_size: System.get_env("SPANDEX_BATCH_SIZE") || 100,
+      sync_threshold: System.get_env("SPANDEX_SYNC_THRESHOLD") || 100,
+      http: HTTPoison
+    ]
+  end
 end

apps/explorer/lib/explorer/chain/token_transfer.ex

@@ -135,6 +135,10 @@ defmodule Explorer.Chain.TokenTransfer do
 
   def page_token_transfer(query, %PagingOptions{key: nil}), do: query
 
+  def page_token_transfer(query, %PagingOptions{key: {token_id}}) do
+    where(query, [token_transfer], token_transfer.token_id > ^token_id)
+  end
+
   def page_token_transfer(query, %PagingOptions{key: {block_number, log_index}}) do
     where(
       query,

apps/explorer/lib/explorer/graphql.ex

@@ -5,13 +5,28 @@ defmodule Explorer.GraphQL do
 
   import Ecto.Query,
     only: [
+      from: 2,
       order_by: 3,
       or_where: 3,
       where: 3
     ]
 
-  alias Explorer.Chain.{Address, Hash, Transaction}
+  alias Explorer.Chain.{
+    Address,
+    Hash,
+    InternalTransaction,
+    Transaction
+  }
+
+  alias Explorer.{Chain, Repo}
+
+  @doc """
+  Returns a query to fetch transactions with a matching `to_address_hash`,
+  `from_address_hash`, or `created_contract_address_hash` field for a given address.
 
+  Orders transactions by descending block number and index.
+  """
+  @spec address_to_transactions_query(Address.t()) :: Ecto.Query.t()
   def address_to_transactions_query(%Address{hash: %Hash{byte_count: unquote(Hash.Address.byte_count())} = address_hash}) do
     Transaction
     |> order_by([transaction], desc: transaction.block_number, desc: transaction.index)
@@ -19,4 +34,37 @@ defmodule Explorer.GraphQL do
     |> or_where([transaction], transaction.from_address_hash == ^address_hash)
     |> or_where([transaction], transaction.created_contract_address_hash == ^address_hash)
   end
+
+  @doc """
+  Returns an internal transaction for a given transaction hash and index.
+  """
+  @spec get_internal_transaction(map()) :: {:ok, InternalTransaction.t()} | {:error, String.t()}
+  def get_internal_transaction(%{transaction_hash: _, index: _} = clauses) do
+    if internal_transaction = Repo.get_by(InternalTransaction, clauses) do
+      {:ok, internal_transaction}
+    else
+      {:error, "Internal transaction not found."}
+    end
+  end
+
+  @doc """
+  Returns a query to fetch internal transactions for a given transaction.
+
+  Orders internal transactions by ascending index.
+  """
+  @spec transaction_to_internal_transactions_query(Transaction.t()) :: Ecto.Query.t()
+  def transaction_to_internal_transactions_query(%Transaction{
+        hash: %Hash{byte_count: unquote(Hash.Full.byte_count())} = hash
+      }) do
+    query =
+      from(
+        it in InternalTransaction,
+        inner_join: t in assoc(it, :transaction),
+        order_by: [asc: it.index],
+        where: it.transaction_hash == ^hash,
+        select: it
+      )
+
+    Chain.where_transaction_has_multiple_internal_transactions(query)
+  end
 end

apps/explorer/lib/explorer/tracer.ex

@@ -0,0 +1,5 @@
+defmodule Explorer.Tracer do
+  @moduledoc false
+
+  use Spandex.Tracer, otp_app: :explorer
+end

apps/explorer/mix.exs

@@ -96,6 +96,12 @@ defmodule Explorer.Mixfile do
       # bypass optional dependency
       {:plug_cowboy, "~> 1.0", only: :test},
       {:sobelow, ">= 0.7.0", only: [:dev, :test], runtime: false},
+      # Tracing
+      {:spandex, github: "spandex-project/spandex", branch: "allow-setting-trace-key", override: true},
+      # `:spandex` integration with Datadog
+      {:spandex_datadog, "~> 0.3.1"},
+      # `:spandex` tracing of `:ecto`
+      {:spandex_ecto, "~> 0.4.0"},
       {:timex, "~> 3.4"},
       # `Timex.Duration` for `Explorer.Chain.average_block_time/0`
       {:timex_ecto, "~> 3.3"}

apps/explorer/test/explorer/chain_test.exs

@@ -3131,7 +3131,7 @@ defmodule Explorer.ChainTest do
           token_id: 29
         )
 
-      paging_options = %PagingOptions{key: {first_page.block_number, first_page.log_index}, page_size: 1}
+      paging_options = %PagingOptions{key: {first_page.token_id}, page_size: 1}
 
       unique_tokens_ids_paginated =
         token_contract_address.hash

apps/explorer/test/explorer/graphql_test.exs

@@ -87,4 +87,98 @@ defmodule Explorer.GraphQLTest do
       assert block_number_and_index_order == Enum.sort(block_number_and_index_order, &(&1 >= &2))
     end
   end
+
+  describe "get_internal_transaction/1" do
+    test "returns existing internal transaction" do
+      transaction = insert(:transaction)
+
+      internal_transaction = insert(:internal_transaction, transaction: transaction, index: 0)
+
+      clauses = %{transaction_hash: transaction.hash, index: internal_transaction.index}
+
+      {:ok, found_internal_transaction} = GraphQL.get_internal_transaction(clauses)
+
+      assert found_internal_transaction.transaction_hash == transaction.hash
+      assert found_internal_transaction.index == internal_transaction.index
+    end
+
+    test "returns error tuple for non-existent internal transaction" do
+      transaction = build(:transaction)
+
+      internal_transaction = build(:internal_transaction, transaction: transaction, index: 0)
+
+      clauses = %{transaction_hash: transaction.hash, index: internal_transaction.index}
+
+      assert GraphQL.get_internal_transaction(clauses) == {:error, "Internal transaction not found."}
+    end
+  end
+
+  describe "transcation_to_internal_transactions_query/1" do
+    test "with transaction with one internal transaction" do
+      transaction1 = insert(:transaction)
+      transaction2 = insert(:transaction)
+
+      internal_transaction = insert(:internal_transaction_create, transaction: transaction1, index: 0)
+      insert(:internal_transaction_create, transaction: transaction2, index: 0)
+
+      [found_internal_transaction] =
+        transaction1
+        |> GraphQL.transaction_to_internal_transactions_query()
+        |> Repo.all()
+
+      assert found_internal_transaction.transaction_hash == transaction1.hash
+      assert found_internal_transaction.index == internal_transaction.index
+    end
+
+    test "with transaction with multiple internal transactions" do
+      transaction1 = insert(:transaction)
+      transaction2 = insert(:transaction)
+
+      for index <- 0..2 do
+        insert(:internal_transaction_create, transaction: transaction1, index: index)
+      end
+
+      insert(:internal_transaction_create, transaction: transaction2, index: 0)
+
+      found_internal_transactions =
+        transaction1
+        |> GraphQL.transaction_to_internal_transactions_query()
+        |> Repo.all()
+
+      assert length(found_internal_transactions) == 3
+
+      for found_internal_transaction <- found_internal_transactions do
+        assert found_internal_transaction.transaction_hash == transaction1.hash
+      end
+    end
+
+    test "orders internal transactions by ascending index" do
+      transaction = insert(:transaction)
+
+      insert(:internal_transaction_create, transaction: transaction, index: 2)
+      insert(:internal_transaction_create, transaction: transaction, index: 0)
+      insert(:internal_transaction_create, transaction: transaction, index: 1)
+
+      found_internal_transactions =
+        transaction
+        |> GraphQL.transaction_to_internal_transactions_query()
+        |> Repo.all()
+
+      index_order = Enum.map(found_internal_transactions, & &1.index)
+
+      assert index_order == Enum.sort(index_order)
+    end
+
+    # Note that `transaction_to_internal_transactions_query/1` relies on
+    # `Explorer.Chain.where_transaction_has_multiple_transactions/1` to ensure the
+    # following behavior:
+    #
+    # * exclude internal transactions of type call with no siblings in the
+    #   transaction
+    #
+    # * include internal transactions of type create, reward, or suicide
+    #   even when they are alone in the parent transaction
+    #
+    # These two requirements are tested in `Explorer.ChainTest`.
+  end
 end

apps/indexer/config/config.exs

@@ -11,6 +11,11 @@ config :indexer,
   # bytes
   memory_limit: 1 <<< 30
 
+config :indexer, Indexer.Tracer,
+  service: :indexer,
+  adapter: SpandexDatadog.Adapter,
+  trace_key: :blockscout
+
 config :logger, :indexer,
   # keep synced with `config/config.exs`
   format: "$time $metadata[$level] $message\n",

apps/indexer/config/dev.exs

@@ -1,5 +1,7 @@
 use Mix.Config
 
+config :indexer, Indexer.Tracer, env: "dev", disabled?: true
+
 config :logger, :indexer,
   level: :debug,
   path: Path.absname("logs/dev/indexer.log")

apps/indexer/config/dev/ganache.exs

@@ -9,7 +9,7 @@ config :indexer,
       url: System.get_env("ETHEREUM_JSONRPC_HTTP_URL") || "http://localhost:7545",
       http_options: [recv_timeout: 60_000, timeout: 60_000, hackney: [pool: :ethereum_jsonrpc]]
     ],
-    variant: EthereumJSONRPC.Geth
+    variant: EthereumJSONRPC.Ganache
   ],
   subscribe_named_arguments: [
     transport: EthereumJSONRPC.WebSocket,

apps/indexer/config/prod.exs

@@ -1,5 +1,7 @@
 use Mix.Config
 
+config :indexer, Indexer.Tracer, env: "production", disabled?: true
+
 config :logger, :indexer,
   level: :info,
   path: Path.absname("logs/prod/indexer.log"),

apps/indexer/config/prod/ganache.exs

@@ -9,7 +9,7 @@ config :indexer,
       url: System.get_env("ETHEREUM_JSONRPC_HTTP_URL") || "http://localhost:7545",
       http_options: [recv_timeout: 60_000, timeout: 60_000, hackney: [pool: :ethereum_jsonrpc]]
     ],
-    variant: EthereumJSONRPC.Geth
+    variant: EthereumJSONRPC.Ganache
   ],
   subscribe_named_arguments: [
     transport: EthereumJSONRPC.WebSocket,

apps/indexer/config/test.exs

@@ -1,5 +1,7 @@
 use Mix.Config
 
+config :indexer, Indexer.Tracer, disabled?: false
+
 config :logger, :indexer,
   level: :warn,
   path: Path.absname("logs/test/indexer.log")

apps/indexer/lib/indexer/block/catchup/fetcher.ex

@@ -3,13 +3,15 @@ defmodule Indexer.Block.Catchup.Fetcher do
   Fetches and indexes block ranges from the block before the latest block to genesis (0) that are missing.
   """
 
+  use Spandex.Decorators
+
   require Logger
 
   import Indexer.Block.Fetcher,
     only: [async_import_coin_balances: 2, async_import_tokens: 1, async_import_uncles: 1, fetch_and_import_range: 2]
 
   alias Explorer.Chain
-  alias Indexer.{Block, InternalTransaction, Sequence, TokenBalance}
+  alias Indexer.{Block, InternalTransaction, Sequence, TokenBalance, Tracer}
   alias Indexer.Memory.Shrinkable
 
   @behaviour Block.Fetcher
@@ -160,6 +162,11 @@ defmodule Indexer.Block.Catchup.Fetcher do
   end
 
   # Run at state.blocks_concurrency max_concurrency when called by `stream_import/1`
+  @decorate trace(
+              name: "fetch",
+              resource: "Indexer.Block.Catchup.Fetcher.fetch_and_import_range_from_sequence/3",
+              tracer: Tracer
+            )
   defp fetch_and_import_range_from_sequence(
          %__MODULE__{block_fetcher: %Block.Fetcher{} = block_fetcher},
          _.._ = range,

apps/indexer/lib/indexer/block/fetcher.ex

@@ -3,11 +3,13 @@ defmodule Indexer.Block.Fetcher do
   Fetches and indexes block ranges.
   """
 
+  use Spandex.Decorators
+
   require Logger
 
   alias EthereumJSONRPC.{Blocks, FetchedBeneficiaries}
   alias Explorer.Chain.{Address, Block, Import}
-  alias Indexer.{AddressExtraction, CoinBalance, MintTransfer, Token, TokenTransfers}
+  alias Indexer.{AddressExtraction, CoinBalance, MintTransfer, Token, TokenTransfers, Tracer}
   alias Indexer.Address.{CoinBalances, TokenBalances}
   alias Indexer.Block.Fetcher.Receipts
   alias Indexer.Block.Transform
@@ -76,6 +78,7 @@ defmodule Indexer.Block.Fetcher do
     struct!(__MODULE__, named_arguments)
   end
 
+  @decorate span(tracer: Tracer)
   @spec fetch_and_import_range(t, Range.t()) ::
           {:ok, %{inserted: %{}, errors: [EthereumJSONRPC.Transport.error()]}}
           | {:error,

apps/indexer/lib/indexer/block/realtime/fetcher.ex

@@ -4,7 +4,9 @@ defmodule Indexer.Block.Realtime.Fetcher do
   """
 
   use GenServer
+  use Spandex.Decorators
 
+  require Indexer.Tracer
   require Logger
 
   import EthereumJSONRPC, only: [integer_to_quantity: 1, quantity_to_integer: 1]
@@ -13,7 +15,7 @@ defmodule Indexer.Block.Realtime.Fetcher do
   alias Ecto.Changeset
   alias EthereumJSONRPC.{FetchedBalances, Subscription}
   alias Explorer.Chain
-  alias Indexer.{AddressExtraction, Block, TokenBalances}
+  alias Indexer.{AddressExtraction, Block, TokenBalances, Tracer}
   alias Indexer.Block.Realtime.TaskSupervisor
 
   @behaviour Block.Fetcher
@@ -125,7 +127,13 @@ defmodule Indexer.Block.Realtime.Fetcher do
     end
   end
 
+  @decorate trace(name: "fetch", resource: "Indexer.Block.Realtime.Fetcher.fetch_and_import_block/3", tracer: Tracer)
   def fetch_and_import_block(block_number_to_fetch, block_fetcher, retry \\ 3) do
+    do_fetch_and_import_block(block_number_to_fetch, block_fetcher, retry)
+  end
+
+  @decorate span(tracer: Tracer)
+  defp do_fetch_and_import_block(block_number_to_fetch, block_fetcher, retry) do
     case fetch_and_import_range(block_fetcher, block_number_to_fetch..block_number_to_fetch) do
       {:ok, %{inserted: _, errors: []}} ->
         Logger.debug(fn ->
@@ -202,7 +210,7 @@ defmodule Indexer.Block.Realtime.Fetcher do
       fetcher = params.block_fetcher
       updated_retry = params.retry - 1
 
-      fetch_and_import_block(number, fetcher, updated_retry)
+      do_fetch_and_import_block(number, fetcher, updated_retry)
     else
       :ignore
     end

apps/indexer/lib/indexer/block/uncle/fetcher.ex

@@ -4,12 +4,14 @@ defmodule Indexer.Block.Uncle.Fetcher do
   `uncle_fetched_at` where the `uncle_hash` matches `hash`.
   """
 
+  use Spandex.Decorators
+
   require Logger
 
   alias EthereumJSONRPC.Blocks
   alias Explorer.Chain
   alias Explorer.Chain.Hash
-  alias Indexer.{AddressExtraction, Block, BufferedTask}
+  alias Indexer.{AddressExtraction, Block, BufferedTask, Tracer}
 
   @behaviour Block.Fetcher
   @behaviour BufferedTask
@@ -66,6 +68,7 @@ defmodule Indexer.Block.Uncle.Fetcher do
   end
 
   @impl BufferedTask
+  @decorate trace(name: "fetch", resource: "Indexer.Block.Uncle.Fetcher.run/2", service: :indexer, tracer: Tracer)
   def run(hashes, %Block.Fetcher{json_rpc_named_arguments: json_rpc_named_arguments} = block_fetcher) do
     # the same block could be included as an uncle on multiple blocks, but we only want to fetch it once
     unique_hashes = Enum.uniq(hashes)

apps/indexer/lib/indexer/coin_balance/fetcher.ex

@@ -4,6 +4,8 @@ defmodule Indexer.CoinBalance.Fetcher do
   `fetched_coin_balance_block_number` to value at max `t:Explorer.Chain.Address.CoinBalance.t/0` `block_number` for the given `t:Explorer.Chain.Address.t/` `hash`.
   """
 
+  use Spandex.Decorators
+
   require Logger
 
   import EthereumJSONRPC, only: [integer_to_quantity: 1, quantity_to_integer: 1]
@@ -11,7 +13,7 @@ defmodule Indexer.CoinBalance.Fetcher do
   alias EthereumJSONRPC.FetchedBalances
   alias Explorer.Chain
   alias Explorer.Chain.{Block, Hash}
-  alias Indexer.BufferedTask
+  alias Indexer.{BufferedTask, Tracer}
 
   @behaviour BufferedTask
 
@@ -65,6 +67,7 @@ defmodule Indexer.CoinBalance.Fetcher do
   end
 
   @impl BufferedTask
+  @decorate trace(name: "fetch", resource: "Indexer.CoinBalance.Fetcher.run/2", service: :indexer, tracer: Tracer)
   def run(entries, json_rpc_named_arguments) do
     # the same address may be used more than once in the same block, but we only want one `Balance` for a given
     # `{address, block}`, so take unique params only

apps/indexer/lib/indexer/internal_transaction/fetcher.ex

@@ -5,13 +5,15 @@ defmodule Indexer.InternalTransaction.Fetcher do
   See `async_fetch/1` for details on configuring limits.
   """
 
+  use Spandex.Decorators
+
   require Logger
 
   import Indexer.Block.Fetcher, only: [async_import_coin_balances: 2]
 
   alias Explorer.Chain
-  alias Indexer.{AddressExtraction, BufferedTask}
   alias Explorer.Chain.{Block, Hash}
+  alias Indexer.{AddressExtraction, BufferedTask, Tracer}
 
   @behaviour BufferedTask
 
@@ -91,6 +93,12 @@ defmodule Indexer.InternalTransaction.Fetcher do
   end
 
   @impl BufferedTask
+  @decorate trace(
+              name: "fetch",
+              resource: "Indexer.InternalTransaction.Fetcher.run/2",
+              service: :indexer,
+              tracer: Tracer
+            )
   def run(entries, json_rpc_named_arguments) do
     unique_entries = unique_entries(entries)
 

apps/indexer/lib/indexer/token/fetcher.ex

@@ -3,11 +3,13 @@ defmodule Indexer.Token.Fetcher do
   Fetches information about a token.
   """
 
+  use Spandex.Decorators
+
   alias Explorer.Chain
   alias Explorer.Chain.Hash.Address
   alias Explorer.Chain.Token
   alias Explorer.Token.MetadataRetriever
-  alias Indexer.BufferedTask
+  alias Indexer.{BufferedTask, Tracer}
 
   @behaviour BufferedTask
 
@@ -47,6 +49,7 @@ defmodule Indexer.Token.Fetcher do
   end
 
   @impl BufferedTask
+  @decorate trace(name: "fetch", resource: "Indexer.Token.Fetcher.run/2", service: :indexer, tracer: Tracer)
   def run([token_contract_address], _json_rpc_named_arguments) do
     case Chain.token_from_address_hash(token_contract_address) do
       {:ok, %Token{cataloged: false} = token} ->

apps/indexer/lib/indexer/token_balance/fetcher.ex

@@ -14,11 +14,13 @@ defmodule Indexer.TokenBalance.Fetcher do
   that always raise errors interacting with the Smart Contract.
   """
 
+  use Spandex.Decorators
+
   require Logger
 
   alias Explorer.Chain
   alias Explorer.Chain.Hash
-  alias Indexer.{BufferedTask, TokenBalances}
+  alias Indexer.{BufferedTask, TokenBalances, Tracer}
 
   @behaviour BufferedTask
 
@@ -74,6 +76,7 @@ defmodule Indexer.TokenBalance.Fetcher do
   when reading their balance in the Smart Contract.
   """
   @impl BufferedTask
+  @decorate trace(name: "fetch", resource: "Indexer.TokenBalance.Fetcher.run/2", tracer: Tracer, service: :indexer)
   def run(entries, _json_rpc_named_arguments) do
     result =
       entries

apps/indexer/lib/indexer/token_balances.ex

@@ -3,11 +3,14 @@ defmodule Indexer.TokenBalances do
   Reads Token's balances using Smart Contract functions from the blockchain.
   """
 
+  use Spandex.Decorators, tracer: Indexer.Tracer
+
+  require Indexer.Tracer
   require Logger
 
   alias Explorer.Chain
   alias Explorer.Token.BalanceReader
-  alias Indexer.TokenBalance
+  alias Indexer.{TokenBalance, Tracer}
 
   # The timeout used for each process opened by Task.async_stream/3. Default 15s.
   @task_timeout 15000
@@ -29,14 +32,17 @@ defmodule Indexer.TokenBalances do
   """
   def fetch_token_balances_from_blockchain([]), do: {:ok, []}
 
+  @decorate span(tracer: Tracer)
   def fetch_token_balances_from_blockchain(token_balances, opts \\ []) do
     Logger.debug(fn -> "fetching #{Enum.count(token_balances)} token balances" end)
 
     task_timeout = Keyword.get(opts, :timeout, @task_timeout)
 
+    task_callback = traced_fetch_token_balance_callback(Tracer.current_span())
+
     requested_token_balances =
       token_balances
-      |> Task.async_stream(&fetch_token_balance/1, timeout: task_timeout, on_timeout: :kill_task)
+      |> Task.async_stream(task_callback, timeout: task_timeout, on_timeout: :kill_task)
       |> Stream.map(&format_task_results/1)
       |> Enum.filter(&ignore_killed_task/1)
 
@@ -50,6 +56,23 @@ defmodule Indexer.TokenBalances do
     {:ok, fetched_token_balances}
   end
 
+  defp traced_fetch_token_balance_callback(%Spandex.Span{} = span) do
+    fn balance ->
+      try do
+        Tracer.continue_trace_from_span("traced_fetch_token_balance_callback/1", span)
+
+        fetch_token_balance(balance)
+      after
+        Tracer.finish_trace()
+      end
+    end
+  end
+
+  defp traced_fetch_token_balance_callback(_) do
+    &fetch_token_balance/1
+  end
+
+  @decorate span(tracer: Tracer)
   defp fetch_token_balance(
          %{
            token_contract_address_hash: token_contract_address_hash,

apps/indexer/lib/indexer/tracer.ex

@@ -0,0 +1,5 @@
+defmodule Indexer.Tracer do
+  @moduledoc false
+
+  use Spandex.Tracer, otp_app: :indexer
+end

apps/indexer/mix.exs

@@ -44,6 +44,8 @@ defmodule Indexer.MixProject do
   # Run "mix help deps" to learn about dependencies.
   defp deps do
     [
+      # Optional dependency of `:spandex` for `Spandex.Decorators`
+      {:decorator, "~> 1.2"},
       # JSONRPC access to Parity for `Explorer.Indexer`
       {:ethereum_jsonrpc, in_umbrella: true},
       # RLP encoding
@@ -57,7 +59,11 @@ defmodule Indexer.MixProject do
       # Log errors and application output to separate files
       {:logger_file_backend, "~> 0.0.10"},
       # Mocking `EthereumJSONRPC.Transport`, so we avoid hitting real chains for local testing
-      {:mox, "~> 0.4", only: [:test]}
+      {:mox, "~> 0.4", only: [:test]},
+      # Tracing
+      {:spandex, github: "spandex-project/spandex", branch: "allow-setting-trace-key", override: true},
+      # `:spandex` integration with Datadog
+      {:spandex_datadog, "~> 0.3.1"}
     ]
   end
 

mix.exs

@@ -60,12 +60,12 @@ defmodule BlockScout.Mixfile do
   # and cannot be accessed from applications inside the apps folder
   defp deps do
     [
+      # Release
+      {:distillery, "~> 2.0", runtime: false},
       # Documentation
       {:ex_doc, "~> 0.19.0", only: [:dev]},
       # Code coverage
-      {:excoveralls, "~> 0.10.0", only: [:test], github: "KronicDeth/excoveralls", branch: "circle-workflows"},
-      # Release
-      {:distillery, "~> 2.0", runtime: false}
+      {:excoveralls, "~> 0.10.0", only: [:test], github: "KronicDeth/excoveralls", branch: "circle-workflows"}
     ]
   end
 end

mix.lock

@@ -24,6 +24,7 @@
   "dataloader": {:hex, :dataloader, "1.0.4", "7c2345c53c9e5b61420013fc53c8463ba347a938b61f66677eb47d9c4a53ac5d", [:mix], [{:ecto, ">= 0.0.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"},
   "db_connection": {:hex, :db_connection, "1.1.3", "89b30ca1ef0a3b469b1c779579590688561d586694a3ce8792985d4d7e575a61", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, optional: false]}, {:poolboy, "~> 1.5", [hex: :poolboy, optional: true]}, {:sbroker, "~> 1.0", [hex: :sbroker, optional: true]}]},
   "decimal": {:hex, :decimal, "1.5.0", "b0433a36d0e2430e3d50291b1c65f53c37d56f83665b43d79963684865beab68", [:mix], []},
+  "decorator": {:hex, :decorator, "1.2.4", "31dfff6143d37f0b68d0bffb3b9f18ace14fea54d4f1b5e4f86ead6f00d9ff6e", [:mix], [], "hexpm"},
   "deep_merge": {:hex, :deep_merge, "0.2.0", "c1050fa2edf4848b9f556fba1b75afc66608a4219659e3311d9c9427b5b680b3", [:mix], [], "hexpm"},
   "dialyxir": {:hex, :dialyxir, "0.5.1", "b331b091720fd93e878137add264bac4f644e1ddae07a70bf7062c7862c4b952", [:mix], []},
   "distillery": {:hex, :distillery, "2.0.12", "6e78fe042df82610ac3fa50bd7d2d8190ad287d120d3cd1682d83a44e8b34dfb", [:mix], [{:artificery, "~> 0.2", [hex: :artificery, repo: "hexpm", optional: false]}], "hexpm"},
@@ -62,13 +63,15 @@
   "math": {:hex, :math, "0.3.0", "e14e7291115201cb155a3567e66d196bf5088a6f55b030d598107d7ae934a11c", [:mix], []},
   "meck": {:hex, :meck, "0.8.12", "1f7b1a9f5d12c511848fec26bbefd09a21e1432eadb8982d9a8aceb9891a3cf2", [:rebar3], [], "hexpm"},
   "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], []},
-  "mime": {:hex, :mime, "1.3.0", "5e8d45a39e95c650900d03f897fbf99ae04f60ab1daa4a34c7a20a5151b7a5fe", [:mix], [], "hexpm"},
+  "mime": {:hex, :mime, "1.3.1", "30ce04ab3175b6ad0bdce0035cba77bba68b813d523d1aac73d9781b4d193cf8", [:mix], [], "hexpm"},
   "mimerl": {:hex, :mimerl, "1.0.2", "993f9b0e084083405ed8252b99460c4f0563e41729ab42d9074fd5e52439be88", [:rebar3], []},
   "mix_erlang_tasks": {:hex, :mix_erlang_tasks, "0.1.0", "36819fec60b80689eb1380938675af215565a89320a9e29c72c70d97512e4649", [:mix], [], "hexpm"},
   "mochiweb": {:hex, :mochiweb, "2.18.0", "eb55f1db3e6e960fac4e6db4e2db9ec3602cc9f30b86cd1481d56545c3145d2e", [:rebar3], [], "hexpm"},
   "mock": {:hex, :mock, "0.3.2", "e98e998fd76c191c7e1a9557c8617912c53df3d4a6132f561eb762b699ef59fa", [:mix], [{:meck, "~> 0.8.8", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm"},
   "mox": {:hex, :mox, "0.4.0", "7f120840f7d626184a3d65de36189ca6f37d432e5d63acd80045198e4c5f7e6e", [:mix], [], "hexpm"},
+  "msgpax": {:hex, :msgpax, "1.1.0", "e31625e256db2decca1ae2b841f21b4d2483b1332649ce3ebc96c7ff7a4986e3", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm"},
   "nimble_parsec": {:hex, :nimble_parsec, "0.4.0", "ee261bb53214943679422be70f1658fff573c5d0b0a1ecd0f18738944f818efe", [:mix], [], "hexpm"},
+  "optimal": {:hex, :optimal, "0.3.6", "46bbf52fbbbd238cda81e02560caa84f93a53c75620f1fe19e81e4ae7b07d1dd", [:mix], [], "hexpm"},
   "parallel_stream": {:hex, :parallel_stream, "1.0.6", "b967be2b23f0f6787fab7ed681b4c45a215a81481fb62b01a5b750fa8f30f76c", [:mix], []},
   "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm"},
   "phoenix": {:hex, :phoenix, "1.3.4", "aaa1b55e5523083a877bcbe9886d9ee180bf2c8754905323493c2ac325903dc5", [:mix], [{:cowboy, "~> 1.0", [hex: :cowboy, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.0", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.3.3 or ~> 1.4", [hex: :plug, repo: "hexpm", optional: false]}, {:poison, "~> 2.2 or ~> 3.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm"},
@@ -92,6 +95,10 @@
   "ranch": {:hex, :ranch, "1.3.2", "e4965a144dc9fbe70e5c077c65e73c57165416a901bd02ea899cfd95aa890986", [:rebar3], []},
   "set_locale": {:git, "https://github.com/minifast/set_locale.git", "da9ae029642bc0fbd9212c2aaf86c0adca70c084", [branch: "master"]},
   "sobelow": {:hex, :sobelow, "0.7.1", "01a52ea8a19be0aa41ce969746f057e90f3994f1607c771968359718bd0e6988", [:mix], [], "hexpm"},
+  "spandex": {:git, "https://github.com/spandex-project/spandex.git", "92992b4aaf3d92e031c2417ff2e6c9e94d27fe36", [branch: "allow-setting-trace-key"]},
+  "spandex_datadog": {:hex, :spandex_datadog, "0.3.1", "984d27ad1f45cfd243509692f0f63b900a23b79566c529a644c7f3a2b4120603", [:mix], [{:msgpax, "~> 1.1", [hex: :msgpax, repo: "hexpm", optional: false]}, {:spandex, "~> 2.3", [hex: :spandex, repo: "hexpm", optional: false]}], "hexpm"},
+  "spandex_ecto": {:hex, :spandex_ecto, "0.4.0", "deaeaddc11a35f1c551206c53d09bb93a0da5808dbef751430e465c8c7de01d3", [:mix], [{:spandex, "~> 2.2", [hex: :spandex, repo: "hexpm", optional: false]}], "hexpm"},
+  "spandex_phoenix": {:hex, :spandex_phoenix, "0.3.0", "48b0a426bbe4eea3e579bbea77d5eb5a8d4b83d33c95616f9ba64b3ce2faef6c", [:mix], [{:plug, "~> 1.3", [hex: :plug, repo: "hexpm", optional: false]}, {:spandex, "~> 2.2", [hex: :spandex, repo: "hexpm", optional: false]}], "hexpm"},
   "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"},
   "timex": {:hex, :timex, "3.4.1", "e63fc1a37453035e534c3febfe9b6b9e18583ec7b37fd9c390efdef97397d70b", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"},
   "timex_ecto": {:hex, :timex_ecto, "3.3.0", "d5bdef09928e7a60f10a0baa47ce653f29b43d6fee87b30b236b216d0e36b98d", [:mix], [{:ecto, "~> 2.2", [hex: :ecto, repo: "hexpm", optional: false]}, {:timex, "~> 3.1", [hex: :timex, repo: "hexpm", optional: false]}], "hexpm"},