From 5d12613f709df9b426f9af00999211219b26c545 Mon Sep 17 00:00:00 2001 From: zachdaniel Date: Wed, 28 Nov 2018 12:24:43 -0500 Subject: [PATCH] feat: support tracing via spandex --- .credo.exs | 5 ++-- README.md | 29 ++++++++++++++++++- apps/block_scout_web/config/config.exs | 9 +++++- apps/block_scout_web/config/dev.exs | 2 ++ apps/block_scout_web/config/prod.exs | 2 ++ apps/block_scout_web/config/test.exs | 2 ++ .../lib/block_scout_web/endpoint.ex | 2 ++ .../lib/block_scout_web/tracer.ex | 5 ++++ apps/block_scout_web/mix.exs | 6 ++++ apps/ethereum_jsonrpc/config/config.exs | 17 +++++++---- apps/ethereum_jsonrpc/config/dev.exs | 2 ++ apps/ethereum_jsonrpc/config/prod.exs | 2 ++ apps/ethereum_jsonrpc/config/test.exs | 10 ++++--- .../ethereum_jsonrpc/request_coordinator.ex | 24 ++++++++++++--- .../lib/ethereum_jsonrpc/tracer.ex | 5 ++++ apps/ethereum_jsonrpc/mix.exs | 4 +++ apps/explorer/config/config.exs | 10 +++++++ apps/explorer/config/dev.exs | 2 ++ apps/explorer/config/prod.exs | 2 ++ apps/explorer/config/test.exs | 4 ++- apps/explorer/lib/explorer/application.ex | 11 +++++++ apps/explorer/lib/explorer/tracer.ex | 5 ++++ apps/explorer/mix.exs | 6 ++++ apps/indexer/config/config.exs | 5 ++++ apps/indexer/config/dev.exs | 2 ++ apps/indexer/config/prod.exs | 2 ++ apps/indexer/config/test.exs | 2 ++ .../lib/indexer/block/catchup/fetcher.ex | 9 +++++- apps/indexer/lib/indexer/block/fetcher.ex | 5 +++- .../lib/indexer/block/realtime/fetcher.ex | 12 ++++++-- .../lib/indexer/block/uncle/fetcher.ex | 5 +++- .../lib/indexer/coin_balance/fetcher.ex | 5 +++- .../indexer/internal_transaction/fetcher.ex | 10 ++++++- apps/indexer/lib/indexer/token/fetcher.ex | 5 +++- .../lib/indexer/token_balance/fetcher.ex | 5 +++- apps/indexer/lib/indexer/token_balances.ex | 27 +++++++++++++++-- apps/indexer/lib/indexer/tracer.ex | 5 ++++ apps/indexer/mix.exs | 8 ++++- mix.exs | 6 ++-- mix.lock | 9 +++++- 40 files changed, 253 insertions(+), 35 deletions(-) create mode 100644 apps/block_scout_web/lib/block_scout_web/tracer.ex create mode 100644 apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/tracer.ex create mode 100644 apps/explorer/lib/explorer/tracer.ex create mode 100644 apps/indexer/lib/indexer/tracer.ex diff --git a/.credo.exs b/.credo.exs index ceb9421811..7dc6c59a95 100644 --- a/.credo.exs +++ b/.credo.exs @@ -75,8 +75,9 @@ # Priority values are: `low, normal, high, higher` # {Credo.Check.Design.AliasUsage, - excluded_namespaces: ~w(Block Blocks Import Socket Task), - excluded_lastnames: ~w(Address DateTime Exporter Fetcher Full Instrumenter Monitor Name Number Repo Time Unit), + excluded_namespaces: ~w(Block Blocks Import Socket SpandexDatadog Task), + excluded_lastnames: + ~w(Address DateTime Exporter Fetcher Full Instrumenter Monitor Name Number Repo Spec Time Unit), priority: :low}, # For some checks, you can also set other parameters diff --git a/README.md b/README.md index f201753977..2d10d4ce0f 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@

BlockScout

Blockchain Explorer for inspecting and analyzing EVM Chains.

- + [![CircleCI](https://circleci.com/gh/poanetwork/blockscout.svg?style=svg&circle-token=f8823a3d0090407c11f87028c73015a331dbf604)](https://circleci.com/gh/poanetwork/blockscout) [![Coverage Status](https://coveralls.io/repos/github/poanetwork/blockscout/badge.svg?branch=master)](https://coveralls.io/github/poanetwork/blockscout?branch=master) [![Join the chat at https://gitter.im/poanetwork/blockscout](https://badges.gitter.im/poanetwork/blockscout.svg)](https://gitter.im/poanetwork/blockscout?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
@@ -283,6 +283,33 @@ BlockScout is setup to export [Prometheus](https://prometheus.io/) metrics at `/ 3. Click "Load" 6. View the dashboards. (You will need to click-around and use BlockScout for the web-related metrics to show up.) +## Tracing + +Blockscout supports tracing via +[Spandex](http://git@github.com:spandex-project/spandex.git). Each application +has its own tracer, that is configured internally to that application. In order +to enable it, visit each application's `config/.ex` and update its tracer +configuration to change `disabled?: true` to `disabled?: false`. Do this for +each application you'd like included in your trace data. + +Currently, only [Datadog](https://www.datadoghq.com/) is supported as a +tracing backend, but more will be added soon. + +### DataDog + +If you would like to use DataDog, after enabling `Spandex`, set +`"DATADOG_HOST"` and `"DATADOG_PORT"` environment variables to the +host/port that your Datadog agent is running on. For more information on +Datadog and the Datadog agent, see their +[documentation](https://docs.datadoghq.com/). + +### Other + +If you want to use a different backend, remove the +`SpandexDatadog.ApiServer` `Supervisor.child_spec` from +`Explorer.Application` and follow any instructions provided in `Spandex` +for setting up that backend. + ## Memory Usage The work queues for building the index of all blocks, balances (coin and token), and internal transactions can grow quite large. By default, the soft-limit is 1 GiB, which can be changed in `apps/indexer/config/config.exs`: diff --git a/apps/block_scout_web/config/config.exs b/apps/block_scout_web/config/config.exs index 3f3eecb4e1..05b7a3f1a5 100644 --- a/apps/block_scout_web/config/config.exs +++ b/apps/block_scout_web/config/config.exs @@ -18,7 +18,7 @@ config :block_scout_web, BlockScoutWeb.Chain, # Configures the endpoint config :block_scout_web, BlockScoutWeb.Endpoint, - instrumenters: [BlockScoutWeb.Prometheus.Instrumenter], + instrumenters: [BlockScoutWeb.Prometheus.Instrumenter, SpandexPhoenix.Instrumenter], url: [ host: "localhost", path: System.get_env("NETWORK_PATH") || "/" @@ -26,6 +26,11 @@ config :block_scout_web, BlockScoutWeb.Endpoint, render_errors: [view: BlockScoutWeb.ErrorView, accepts: ~w(html json)], pubsub: [name: BlockScoutWeb.PubSub, adapter: Phoenix.PubSub.PG2] +config :block_scout_web, BlockScoutWeb.Tracer, + service: :block_scout_web, + adapter: SpandexDatadog.Adapter, + trace_key: :blockscout + # Configures gettext config :block_scout_web, BlockScoutWeb.Gettext, locales: ~w(en), default_locale: "en" @@ -46,6 +51,8 @@ config :logger, :block_scout_web, metadata: [:application, :request_id], metadata_filter: [application: :block_scout_web] +config :spandex_phoenix, tracer: BlockScoutWeb.Tracer + config :wobserver, # return only the local node discovery: :none, diff --git a/apps/block_scout_web/config/dev.exs b/apps/block_scout_web/config/dev.exs index d821ac555f..2b50183667 100644 --- a/apps/block_scout_web/config/dev.exs +++ b/apps/block_scout_web/config/dev.exs @@ -48,6 +48,8 @@ config :block_scout_web, BlockScoutWeb.Endpoint, ] ] +config :block_scout_web, BlockScoutWeb.Tracer, env: "dev", disabled?: true + config :logger, :block_scout_web, level: :debug, path: Path.absname("logs/dev/block_scout_web.log") diff --git a/apps/block_scout_web/config/prod.exs b/apps/block_scout_web/config/prod.exs index a6de66fea4..1fb27bdbed 100644 --- a/apps/block_scout_web/config/prod.exs +++ b/apps/block_scout_web/config/prod.exs @@ -24,6 +24,8 @@ config :block_scout_web, BlockScoutWeb.Endpoint, port: System.get_env("PORT") ] +config :block_scout_web, BlockScoutWeb.Tracer, env: "production", disabled?: true + config :logger, :block_scout_web, level: :info, path: Path.absname("logs/prod/block_scout_web.log"), diff --git a/apps/block_scout_web/config/test.exs b/apps/block_scout_web/config/test.exs index 74c119ab45..531ea3f1e6 100644 --- a/apps/block_scout_web/config/test.exs +++ b/apps/block_scout_web/config/test.exs @@ -9,6 +9,8 @@ config :block_scout_web, BlockScoutWeb.Endpoint, secret_key_base: "27Swe6KtEtmN37WyEYRjKWyxYULNtrxlkCEKur4qoV+Lwtk8lafsR16ifz1XBBYj", server: true +config :block_scout_web, BlockScoutWeb.Tracer, disabled?: false + config :logger, :block_scout_web, level: :warn, path: Path.absname("logs/test/block_scout_web.log") diff --git a/apps/block_scout_web/lib/block_scout_web/endpoint.ex b/apps/block_scout_web/lib/block_scout_web/endpoint.ex index 673f3ea7c6..caed03e3ff 100644 --- a/apps/block_scout_web/lib/block_scout_web/endpoint.ex +++ b/apps/block_scout_web/lib/block_scout_web/endpoint.ex @@ -68,6 +68,8 @@ defmodule BlockScoutWeb.Endpoint do signing_salt: "iC2ksJHS" ) + use SpandexPhoenix + plug(BlockScoutWeb.Prometheus.Exporter) plug(BlockScoutWeb.Router) diff --git a/apps/block_scout_web/lib/block_scout_web/tracer.ex b/apps/block_scout_web/lib/block_scout_web/tracer.ex new file mode 100644 index 0000000000..6bca34d564 --- /dev/null +++ b/apps/block_scout_web/lib/block_scout_web/tracer.ex @@ -0,0 +1,5 @@ +defmodule BlockScoutWeb.Tracer do + @moduledoc false + + use Spandex.Tracer, otp_app: :block_scout_web +end diff --git a/apps/block_scout_web/mix.exs b/apps/block_scout_web/mix.exs index d7a1be661e..22a7624add 100644 --- a/apps/block_scout_web/mix.exs +++ b/apps/block_scout_web/mix.exs @@ -112,6 +112,12 @@ defmodule BlockScoutWeb.Mixfile do {:prometheus_process_collector, "~> 1.3"}, {:qrcode, "~> 0.1.0"}, {:sobelow, ">= 0.7.0", only: [:dev, :test], runtime: false}, + # Tracing + {:spandex, github: "spandex-project/spandex", branch: "allow-setting-trace-key", override: true}, + # `:spandex` integration with Datadog + {:spandex_datadog, "~> 0.3.1"}, + # `:spandex` tracing of `:phoenix` + {:spandex_phoenix, "~> 0.3.0"}, {:timex, "~> 3.4"}, {:wallaby, "~> 0.20", only: [:test], runtime: false}, {:wobserver, "~> 0.1.8"} diff --git a/apps/ethereum_jsonrpc/config/config.exs b/apps/ethereum_jsonrpc/config/config.exs index 782c1ae1ff..1725d5abe9 100644 --- a/apps/ethereum_jsonrpc/config/config.exs +++ b/apps/ethereum_jsonrpc/config/config.exs @@ -1,11 +1,5 @@ use Mix.Config -config :logger, :ethereum_jsonrpc, - # keep synced with `config/config.exs` - format: "$time $metadata[$level] $message\n", - metadata: [:application, :request_id], - metadata_filter: [application: :ethereum_jsonrpc] - config :ethereum_jsonrpc, EthereumJSONRPC.RequestCoordinator, rolling_window_opts: [ window_count: 12, @@ -15,6 +9,17 @@ config :ethereum_jsonrpc, EthereumJSONRPC.RequestCoordinator, wait_per_timeout: :timer.seconds(20), max_jitter: :timer.seconds(2) +config :ethereum_jsonrpc, EthereumJSONRPC.Tracer, + service: :ethereum_jsonrpc, + adapter: SpandexDatadog.Adapter, + trace_key: :blockscout + +config :logger, :ethereum_jsonrpc, + # keep synced with `config/config.exs` + format: "$time $metadata[$level] $message\n", + metadata: [:application, :request_id], + metadata_filter: [application: :ethereum_jsonrpc] + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" diff --git a/apps/ethereum_jsonrpc/config/dev.exs b/apps/ethereum_jsonrpc/config/dev.exs index 1e5d0ebbe4..1f81a56cb5 100644 --- a/apps/ethereum_jsonrpc/config/dev.exs +++ b/apps/ethereum_jsonrpc/config/dev.exs @@ -1,5 +1,7 @@ use Mix.Config +config :ethereum_jsonrpc, EthereumJSONRPC.Tracer, env: "dev", disabled?: true + config :logger, :ethereum_jsonrpc, level: :debug, path: Path.absname("logs/dev/ethereum_jsonrpc.log") diff --git a/apps/ethereum_jsonrpc/config/prod.exs b/apps/ethereum_jsonrpc/config/prod.exs index 0f1324c682..4b4d80babb 100644 --- a/apps/ethereum_jsonrpc/config/prod.exs +++ b/apps/ethereum_jsonrpc/config/prod.exs @@ -1,5 +1,7 @@ use Mix.Config +config :ethereum_jsonrpc, EthereumJSONRPC.Tracer, env: "production", disabled?: true + config :logger, :ethereum_jsonrpc, level: :info, path: Path.absname("logs/prod/ethereum_jsonrpc.log"), diff --git a/apps/ethereum_jsonrpc/config/test.exs b/apps/ethereum_jsonrpc/config/test.exs index cf1666a448..e76eab248f 100644 --- a/apps/ethereum_jsonrpc/config/test.exs +++ b/apps/ethereum_jsonrpc/config/test.exs @@ -1,9 +1,5 @@ use Mix.Config -config :logger, :ethereum_jsonrpc, - level: :warn, - path: Path.absname("logs/test/ethereum_jsonrpc.log") - config :ethereum_jsonrpc, EthereumJSONRPC.RequestCoordinator, rolling_window_opts: [ window_count: 3, @@ -12,3 +8,9 @@ config :ethereum_jsonrpc, EthereumJSONRPC.RequestCoordinator, ], wait_per_timeout: 2, max_jitter: 1 + +config :ethereum_jsonrpc, EthereumJSONRPC.Tracer, disabled?: false + +config :logger, :ethereum_jsonrpc, + level: :warn, + path: Path.absname("logs/test/ethereum_jsonrpc.log") diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/request_coordinator.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/request_coordinator.ex index 8e8a767d76..c72953555b 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/request_coordinator.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/request_coordinator.ex @@ -42,7 +42,9 @@ defmodule EthereumJSONRPC.RequestCoordinator do With this configuration, timeouts are tracked for 6 windows of 10 seconds for a total of 1 minute. """ - alias EthereumJSONRPC.{RollingWindow, Transport} + require EthereumJSONRPC.Tracer + + alias EthereumJSONRPC.{RollingWindow, Tracer, Transport} @error_key :throttleable_error_count @@ -63,9 +65,11 @@ defmodule EthereumJSONRPC.RequestCoordinator do if sleep_time <= throttle_timeout do :timer.sleep(sleep_time) - request - |> transport.json_rpc(transport_options) - |> handle_transport_response() + trace_request(request, fn -> + request + |> transport.json_rpc(transport_options) + |> handle_transport_response() + end) else :timer.sleep(throttle_timeout) @@ -73,6 +77,18 @@ defmodule EthereumJSONRPC.RequestCoordinator do end end + defp trace_request([request | _], fun) do + trace_request(request, fun) + end + + defp trace_request(%{method: method}, fun) do + Tracer.span "RequestCoordinator.perform/4", resource: method, service: :ethereum_jsonrpc do + fun.() + end + end + + defp trace_request(_, fun), do: fun.() + defp handle_transport_response({:error, {:bad_gateway, _}} = error) do RollingWindow.inc(table(), @error_key) error diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/tracer.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/tracer.ex new file mode 100644 index 0000000000..159ed37bbb --- /dev/null +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/tracer.ex @@ -0,0 +1,5 @@ +defmodule EthereumJSONRPC.Tracer do + @moduledoc false + + use Spandex.Tracer, otp_app: :ethereum_jsonrpc +end diff --git a/apps/ethereum_jsonrpc/mix.exs b/apps/ethereum_jsonrpc/mix.exs index 02c8ccfe53..d91458c4ab 100644 --- a/apps/ethereum_jsonrpc/mix.exs +++ b/apps/ethereum_jsonrpc/mix.exs @@ -76,6 +76,10 @@ defmodule EthereumJsonrpc.MixProject do {:logger_file_backend, "~> 0.0.10"}, # Mocking `EthereumJSONRPC.Transport` and `EthereumJSONRPC.HTTP` so we avoid hitting real chains for local testing {:mox, "~> 0.4", only: [:test]}, + # Tracing + {:spandex, github: "spandex-project/spandex", branch: "allow-setting-trace-key", override: true}, + # `:spandex` integration with Datadog + {:spandex_datadog, "~> 0.3.1"}, # Convert unix timestamps in JSONRPC to DateTimes {:timex, "~> 3.4"}, # Encode/decode function names and arguments diff --git a/apps/explorer/config/config.exs b/apps/explorer/config/config.exs index a4af332015..b5892e3525 100644 --- a/apps/explorer/config/config.exs +++ b/apps/explorer/config/config.exs @@ -25,6 +25,11 @@ config :explorer, Explorer.Repo, loggers: [Explorer.Repo.PrometheusLogger, Ecto.LogEntry], migration_timestamps: [type: :utc_datetime] +config :explorer, Explorer.Tracer, + service: :explorer, + adapter: SpandexDatadog.Adapter, + trace_key: :blockscout + config :explorer, Explorer.Counters.TokenTransferCounter, enabled: true config :explorer, Explorer.Counters.TokenHoldersCounter, enabled: true, enable_consolidation: true @@ -46,6 +51,11 @@ config :logger, :explorer, metadata: [:application, :request_id], metadata_filter: [application: :explorer] +config :spandex_ecto, SpandexEcto.EctoLogger, + service: :ecto, + tracer: Explorer.Tracer, + otp_app: :explorer + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" diff --git a/apps/explorer/config/dev.exs b/apps/explorer/config/dev.exs index 8cc1f28a5f..2e61bb9124 100644 --- a/apps/explorer/config/dev.exs +++ b/apps/explorer/config/dev.exs @@ -9,6 +9,8 @@ config :explorer, Explorer.Repo, pool_timeout: 60_000, timeout: 80_000 +config :explorer, Explorer.Tracer, env: "dev", disabled?: true + config :logger, :explorer, level: :debug, path: Path.absname("logs/dev/explorer.log") diff --git a/apps/explorer/config/prod.exs b/apps/explorer/config/prod.exs index bc8e95d9db..4d5fe15c46 100644 --- a/apps/explorer/config/prod.exs +++ b/apps/explorer/config/prod.exs @@ -9,6 +9,8 @@ config :explorer, Explorer.Repo, prepare: :unnamed, timeout: 60_000 +config :explorer, Explorer.Tracer, env: "production", disabled?: true + config :logger, :explorer, level: :info, path: Path.absname("logs/prod/explorer.log"), diff --git a/apps/explorer/config/test.exs b/apps/explorer/config/test.exs index 764f2f6ed1..f0e4330a01 100644 --- a/apps/explorer/config/test.exs +++ b/apps/explorer/config/test.exs @@ -3,6 +3,8 @@ use Mix.Config # Lower hashing rounds for faster tests config :bcrypt_elixir, log_rounds: 4 +config :explorer, Explorer.Counters.TokenHoldersCounter, enabled: true, enable_consolidation: false + # Configure your database config :explorer, Explorer.Repo, adapter: Ecto.Adapters.Postgres, @@ -17,7 +19,7 @@ config :explorer, Explorer.ExchangeRates, enabled: false config :explorer, Explorer.Market.History.Cataloger, enabled: false -config :explorer, Explorer.Counters.TokenHoldersCounter, enabled: true, enable_consolidation: false +config :explorer, Explorer.Tracer, disabled?: false config :logger, :explorer, level: :warn, diff --git a/apps/explorer/lib/explorer/application.ex b/apps/explorer/lib/explorer/application.ex index dac4ce0c8a..665c9b08a0 100644 --- a/apps/explorer/lib/explorer/application.ex +++ b/apps/explorer/lib/explorer/application.ex @@ -15,6 +15,7 @@ defmodule Explorer.Application do # Children to start in all environments base_children = [ Explorer.Repo, + Supervisor.Spec.worker(SpandexDatadog.ApiServer, [datadog_opts()]), Supervisor.child_spec({Task.Supervisor, name: Explorer.MarketTaskSupervisor}, id: Explorer.MarketTaskSupervisor), Supervisor.child_spec({Task.Supervisor, name: Explorer.TaskSupervisor}, id: Explorer.TaskSupervisor), {Registry, keys: :duplicate, name: Registry.ChainEvents, id: Registry.ChainEvents}, @@ -52,4 +53,14 @@ defmodule Explorer.Application do [] end end + + defp datadog_opts do + [ + host: System.get_env("DATADOG_HOST") || "localhost", + port: System.get_env("DATADOG_PORT") || 8126, + batch_size: System.get_env("SPANDEX_BATCH_SIZE") || 100, + sync_threshold: System.get_env("SPANDEX_SYNC_THRESHOLD") || 100, + http: HTTPoison + ] + end end diff --git a/apps/explorer/lib/explorer/tracer.ex b/apps/explorer/lib/explorer/tracer.ex new file mode 100644 index 0000000000..20297d80ab --- /dev/null +++ b/apps/explorer/lib/explorer/tracer.ex @@ -0,0 +1,5 @@ +defmodule Explorer.Tracer do + @moduledoc false + + use Spandex.Tracer, otp_app: :explorer +end diff --git a/apps/explorer/mix.exs b/apps/explorer/mix.exs index 90da255dd0..e7b0a6263f 100644 --- a/apps/explorer/mix.exs +++ b/apps/explorer/mix.exs @@ -96,6 +96,12 @@ defmodule Explorer.Mixfile do # bypass optional dependency {:plug_cowboy, "~> 1.0", only: :test}, {:sobelow, ">= 0.7.0", only: [:dev, :test], runtime: false}, + # Tracing + {:spandex, github: "spandex-project/spandex", branch: "allow-setting-trace-key", override: true}, + # `:spandex` integration with Datadog + {:spandex_datadog, "~> 0.3.1"}, + # `:spandex` tracing of `:ecto` + {:spandex_ecto, "~> 0.4.0"}, {:timex, "~> 3.4"}, # `Timex.Duration` for `Explorer.Chain.average_block_time/0` {:timex_ecto, "~> 3.3"} diff --git a/apps/indexer/config/config.exs b/apps/indexer/config/config.exs index e19bcd3610..0225befbf3 100644 --- a/apps/indexer/config/config.exs +++ b/apps/indexer/config/config.exs @@ -11,6 +11,11 @@ config :indexer, # bytes memory_limit: 1 <<< 30 +config :indexer, Indexer.Tracer, + service: :indexer, + adapter: SpandexDatadog.Adapter, + trace_key: :blockscout + config :logger, :indexer, # keep synced with `config/config.exs` format: "$time $metadata[$level] $message\n", diff --git a/apps/indexer/config/dev.exs b/apps/indexer/config/dev.exs index 337e2f9b97..ffd1d6b0ee 100644 --- a/apps/indexer/config/dev.exs +++ b/apps/indexer/config/dev.exs @@ -1,5 +1,7 @@ use Mix.Config +config :indexer, Indexer.Tracer, env: "dev", disabled?: true + config :logger, :indexer, level: :debug, path: Path.absname("logs/dev/indexer.log") diff --git a/apps/indexer/config/prod.exs b/apps/indexer/config/prod.exs index f342efa867..49800bcc15 100644 --- a/apps/indexer/config/prod.exs +++ b/apps/indexer/config/prod.exs @@ -1,5 +1,7 @@ use Mix.Config +config :indexer, Indexer.Tracer, env: "production", disabled?: true + config :logger, :indexer, level: :info, path: Path.absname("logs/prod/indexer.log"), diff --git a/apps/indexer/config/test.exs b/apps/indexer/config/test.exs index e1ba98f859..e9c5505405 100644 --- a/apps/indexer/config/test.exs +++ b/apps/indexer/config/test.exs @@ -1,5 +1,7 @@ use Mix.Config +config :indexer, Indexer.Tracer, disabled?: false + config :logger, :indexer, level: :warn, path: Path.absname("logs/test/indexer.log") diff --git a/apps/indexer/lib/indexer/block/catchup/fetcher.ex b/apps/indexer/lib/indexer/block/catchup/fetcher.ex index 79611270aa..56b96e761f 100644 --- a/apps/indexer/lib/indexer/block/catchup/fetcher.ex +++ b/apps/indexer/lib/indexer/block/catchup/fetcher.ex @@ -3,13 +3,15 @@ defmodule Indexer.Block.Catchup.Fetcher do Fetches and indexes block ranges from the block before the latest block to genesis (0) that are missing. """ + use Spandex.Decorators + require Logger import Indexer.Block.Fetcher, only: [async_import_coin_balances: 2, async_import_tokens: 1, async_import_uncles: 1, fetch_and_import_range: 2] alias Explorer.Chain - alias Indexer.{Block, InternalTransaction, Sequence, TokenBalance} + alias Indexer.{Block, InternalTransaction, Sequence, TokenBalance, Tracer} alias Indexer.Memory.Shrinkable @behaviour Block.Fetcher @@ -160,6 +162,11 @@ defmodule Indexer.Block.Catchup.Fetcher do end # Run at state.blocks_concurrency max_concurrency when called by `stream_import/1` + @decorate trace( + name: "fetch", + resource: "Indexer.Block.Catchup.Fetcher.fetch_and_import_range_from_sequence/3", + tracer: Tracer + ) defp fetch_and_import_range_from_sequence( %__MODULE__{block_fetcher: %Block.Fetcher{} = block_fetcher}, _.._ = range, diff --git a/apps/indexer/lib/indexer/block/fetcher.ex b/apps/indexer/lib/indexer/block/fetcher.ex index 5e7585b59c..1d48135f6e 100644 --- a/apps/indexer/lib/indexer/block/fetcher.ex +++ b/apps/indexer/lib/indexer/block/fetcher.ex @@ -3,11 +3,13 @@ defmodule Indexer.Block.Fetcher do Fetches and indexes block ranges. """ + use Spandex.Decorators + require Logger alias EthereumJSONRPC.{Blocks, FetchedBeneficiaries} alias Explorer.Chain.{Address, Block, Import} - alias Indexer.{AddressExtraction, CoinBalance, MintTransfer, Token, TokenTransfers} + alias Indexer.{AddressExtraction, CoinBalance, MintTransfer, Token, TokenTransfers, Tracer} alias Indexer.Address.{CoinBalances, TokenBalances} alias Indexer.Block.Fetcher.Receipts alias Indexer.Block.Transform @@ -76,6 +78,7 @@ defmodule Indexer.Block.Fetcher do struct!(__MODULE__, named_arguments) end + @decorate span(tracer: Tracer) @spec fetch_and_import_range(t, Range.t()) :: {:ok, %{inserted: %{}, errors: [EthereumJSONRPC.Transport.error()]}} | {:error, diff --git a/apps/indexer/lib/indexer/block/realtime/fetcher.ex b/apps/indexer/lib/indexer/block/realtime/fetcher.ex index e1667679af..f9a37ef47b 100644 --- a/apps/indexer/lib/indexer/block/realtime/fetcher.ex +++ b/apps/indexer/lib/indexer/block/realtime/fetcher.ex @@ -4,7 +4,9 @@ defmodule Indexer.Block.Realtime.Fetcher do """ use GenServer + use Spandex.Decorators + require Indexer.Tracer require Logger import EthereumJSONRPC, only: [integer_to_quantity: 1, quantity_to_integer: 1] @@ -13,7 +15,7 @@ defmodule Indexer.Block.Realtime.Fetcher do alias Ecto.Changeset alias EthereumJSONRPC.{FetchedBalances, Subscription} alias Explorer.Chain - alias Indexer.{AddressExtraction, Block, TokenBalances} + alias Indexer.{AddressExtraction, Block, TokenBalances, Tracer} alias Indexer.Block.Realtime.TaskSupervisor @behaviour Block.Fetcher @@ -125,7 +127,13 @@ defmodule Indexer.Block.Realtime.Fetcher do end end + @decorate trace(name: "fetch", resource: "Indexer.Block.Realtime.Fetcher.fetch_and_import_block/3", tracer: Tracer) def fetch_and_import_block(block_number_to_fetch, block_fetcher, retry \\ 3) do + do_fetch_and_import_block(block_number_to_fetch, block_fetcher, retry) + end + + @decorate span(tracer: Tracer) + defp do_fetch_and_import_block(block_number_to_fetch, block_fetcher, retry) do case fetch_and_import_range(block_fetcher, block_number_to_fetch..block_number_to_fetch) do {:ok, %{inserted: _, errors: []}} -> Logger.debug(fn -> @@ -202,7 +210,7 @@ defmodule Indexer.Block.Realtime.Fetcher do fetcher = params.block_fetcher updated_retry = params.retry - 1 - fetch_and_import_block(number, fetcher, updated_retry) + do_fetch_and_import_block(number, fetcher, updated_retry) else :ignore end diff --git a/apps/indexer/lib/indexer/block/uncle/fetcher.ex b/apps/indexer/lib/indexer/block/uncle/fetcher.ex index 24e5499a29..07bec5c7cc 100644 --- a/apps/indexer/lib/indexer/block/uncle/fetcher.ex +++ b/apps/indexer/lib/indexer/block/uncle/fetcher.ex @@ -4,12 +4,14 @@ defmodule Indexer.Block.Uncle.Fetcher do `uncle_fetched_at` where the `uncle_hash` matches `hash`. """ + use Spandex.Decorators + require Logger alias EthereumJSONRPC.Blocks alias Explorer.Chain alias Explorer.Chain.Hash - alias Indexer.{AddressExtraction, Block, BufferedTask} + alias Indexer.{AddressExtraction, Block, BufferedTask, Tracer} @behaviour Block.Fetcher @behaviour BufferedTask @@ -66,6 +68,7 @@ defmodule Indexer.Block.Uncle.Fetcher do end @impl BufferedTask + @decorate trace(name: "fetch", resource: "Indexer.Block.Uncle.Fetcher.run/2", service: :indexer, tracer: Tracer) def run(hashes, %Block.Fetcher{json_rpc_named_arguments: json_rpc_named_arguments} = block_fetcher) do # the same block could be included as an uncle on multiple blocks, but we only want to fetch it once unique_hashes = Enum.uniq(hashes) diff --git a/apps/indexer/lib/indexer/coin_balance/fetcher.ex b/apps/indexer/lib/indexer/coin_balance/fetcher.ex index 27240c5aad..54a09e8139 100644 --- a/apps/indexer/lib/indexer/coin_balance/fetcher.ex +++ b/apps/indexer/lib/indexer/coin_balance/fetcher.ex @@ -4,6 +4,8 @@ defmodule Indexer.CoinBalance.Fetcher do `fetched_coin_balance_block_number` to value at max `t:Explorer.Chain.Address.CoinBalance.t/0` `block_number` for the given `t:Explorer.Chain.Address.t/` `hash`. """ + use Spandex.Decorators + require Logger import EthereumJSONRPC, only: [integer_to_quantity: 1, quantity_to_integer: 1] @@ -11,7 +13,7 @@ defmodule Indexer.CoinBalance.Fetcher do alias EthereumJSONRPC.FetchedBalances alias Explorer.Chain alias Explorer.Chain.{Block, Hash} - alias Indexer.BufferedTask + alias Indexer.{BufferedTask, Tracer} @behaviour BufferedTask @@ -65,6 +67,7 @@ defmodule Indexer.CoinBalance.Fetcher do end @impl BufferedTask + @decorate trace(name: "fetch", resource: "Indexer.CoinBalance.Fetcher.run/2", service: :indexer, tracer: Tracer) def run(entries, json_rpc_named_arguments) do # the same address may be used more than once in the same block, but we only want one `Balance` for a given # `{address, block}`, so take unique params only diff --git a/apps/indexer/lib/indexer/internal_transaction/fetcher.ex b/apps/indexer/lib/indexer/internal_transaction/fetcher.ex index 65204a18ee..8af62de419 100644 --- a/apps/indexer/lib/indexer/internal_transaction/fetcher.ex +++ b/apps/indexer/lib/indexer/internal_transaction/fetcher.ex @@ -5,13 +5,15 @@ defmodule Indexer.InternalTransaction.Fetcher do See `async_fetch/1` for details on configuring limits. """ + use Spandex.Decorators + require Logger import Indexer.Block.Fetcher, only: [async_import_coin_balances: 2] alias Explorer.Chain - alias Indexer.{AddressExtraction, BufferedTask} alias Explorer.Chain.{Block, Hash} + alias Indexer.{AddressExtraction, BufferedTask, Tracer} @behaviour BufferedTask @@ -91,6 +93,12 @@ defmodule Indexer.InternalTransaction.Fetcher do end @impl BufferedTask + @decorate trace( + name: "fetch", + resource: "Indexer.InternalTransaction.Fetcher.run/2", + service: :indexer, + tracer: Tracer + ) def run(entries, json_rpc_named_arguments) do unique_entries = unique_entries(entries) diff --git a/apps/indexer/lib/indexer/token/fetcher.ex b/apps/indexer/lib/indexer/token/fetcher.ex index eeb51dbab7..c18f1d6488 100644 --- a/apps/indexer/lib/indexer/token/fetcher.ex +++ b/apps/indexer/lib/indexer/token/fetcher.ex @@ -3,11 +3,13 @@ defmodule Indexer.Token.Fetcher do Fetches information about a token. """ + use Spandex.Decorators + alias Explorer.Chain alias Explorer.Chain.Hash.Address alias Explorer.Chain.Token alias Explorer.Token.MetadataRetriever - alias Indexer.BufferedTask + alias Indexer.{BufferedTask, Tracer} @behaviour BufferedTask @@ -47,6 +49,7 @@ defmodule Indexer.Token.Fetcher do end @impl BufferedTask + @decorate trace(name: "fetch", resource: "Indexer.Token.Fetcher.run/2", service: :indexer, tracer: Tracer) def run([token_contract_address], _json_rpc_named_arguments) do case Chain.token_from_address_hash(token_contract_address) do {:ok, %Token{cataloged: false} = token} -> diff --git a/apps/indexer/lib/indexer/token_balance/fetcher.ex b/apps/indexer/lib/indexer/token_balance/fetcher.ex index d6acf3817c..a57d1a235b 100644 --- a/apps/indexer/lib/indexer/token_balance/fetcher.ex +++ b/apps/indexer/lib/indexer/token_balance/fetcher.ex @@ -14,11 +14,13 @@ defmodule Indexer.TokenBalance.Fetcher do that always raise errors interacting with the Smart Contract. """ + use Spandex.Decorators + require Logger alias Explorer.Chain alias Explorer.Chain.Hash - alias Indexer.{BufferedTask, TokenBalances} + alias Indexer.{BufferedTask, TokenBalances, Tracer} @behaviour BufferedTask @@ -74,6 +76,7 @@ defmodule Indexer.TokenBalance.Fetcher do when reading their balance in the Smart Contract. """ @impl BufferedTask + @decorate trace(name: "fetch", resource: "Indexer.TokenBalance.Fetcher.run/2", tracer: Tracer, service: :indexer) def run(entries, _json_rpc_named_arguments) do result = entries diff --git a/apps/indexer/lib/indexer/token_balances.ex b/apps/indexer/lib/indexer/token_balances.ex index 481ca9b670..a00345138f 100644 --- a/apps/indexer/lib/indexer/token_balances.ex +++ b/apps/indexer/lib/indexer/token_balances.ex @@ -3,11 +3,14 @@ defmodule Indexer.TokenBalances do Reads Token's balances using Smart Contract functions from the blockchain. """ + use Spandex.Decorators, tracer: Indexer.Tracer + + require Indexer.Tracer require Logger alias Explorer.Chain alias Explorer.Token.BalanceReader - alias Indexer.TokenBalance + alias Indexer.{TokenBalance, Tracer} # The timeout used for each process opened by Task.async_stream/3. Default 15s. @task_timeout 15000 @@ -29,14 +32,17 @@ defmodule Indexer.TokenBalances do """ def fetch_token_balances_from_blockchain([]), do: {:ok, []} + @decorate span(tracer: Tracer) def fetch_token_balances_from_blockchain(token_balances, opts \\ []) do Logger.debug(fn -> "fetching #{Enum.count(token_balances)} token balances" end) task_timeout = Keyword.get(opts, :timeout, @task_timeout) + task_callback = traced_fetch_token_balance_callback(Tracer.current_span()) + requested_token_balances = token_balances - |> Task.async_stream(&fetch_token_balance/1, timeout: task_timeout, on_timeout: :kill_task) + |> Task.async_stream(task_callback, timeout: task_timeout, on_timeout: :kill_task) |> Stream.map(&format_task_results/1) |> Enum.filter(&ignore_killed_task/1) @@ -50,6 +56,23 @@ defmodule Indexer.TokenBalances do {:ok, fetched_token_balances} end + defp traced_fetch_token_balance_callback(%Spandex.Span{} = span) do + fn balance -> + try do + Tracer.continue_trace_from_span("traced_fetch_token_balance_callback/1", span) + + fetch_token_balance(balance) + after + Tracer.finish_trace() + end + end + end + + defp traced_fetch_token_balance_callback(_) do + &fetch_token_balance/1 + end + + @decorate span(tracer: Tracer) defp fetch_token_balance( %{ token_contract_address_hash: token_contract_address_hash, diff --git a/apps/indexer/lib/indexer/tracer.ex b/apps/indexer/lib/indexer/tracer.ex new file mode 100644 index 0000000000..7e123b99cd --- /dev/null +++ b/apps/indexer/lib/indexer/tracer.ex @@ -0,0 +1,5 @@ +defmodule Indexer.Tracer do + @moduledoc false + + use Spandex.Tracer, otp_app: :indexer +end diff --git a/apps/indexer/mix.exs b/apps/indexer/mix.exs index 44b5d3dea5..ad4807a767 100644 --- a/apps/indexer/mix.exs +++ b/apps/indexer/mix.exs @@ -44,6 +44,8 @@ defmodule Indexer.MixProject do # Run "mix help deps" to learn about dependencies. defp deps do [ + # Optional dependency of `:spandex` for `Spandex.Decorators` + {:decorator, "~> 1.2"}, # JSONRPC access to Parity for `Explorer.Indexer` {:ethereum_jsonrpc, in_umbrella: true}, # RLP encoding @@ -57,7 +59,11 @@ defmodule Indexer.MixProject do # Log errors and application output to separate files {:logger_file_backend, "~> 0.0.10"}, # Mocking `EthereumJSONRPC.Transport`, so we avoid hitting real chains for local testing - {:mox, "~> 0.4", only: [:test]} + {:mox, "~> 0.4", only: [:test]}, + # Tracing + {:spandex, github: "spandex-project/spandex", branch: "allow-setting-trace-key", override: true}, + # `:spandex` integration with Datadog + {:spandex_datadog, "~> 0.3.1"} ] end diff --git a/mix.exs b/mix.exs index f76a8cfe00..70dea50bd4 100644 --- a/mix.exs +++ b/mix.exs @@ -60,12 +60,12 @@ defmodule BlockScout.Mixfile do # and cannot be accessed from applications inside the apps folder defp deps do [ + # Release + {:distillery, "~> 2.0", runtime: false}, # Documentation {:ex_doc, "~> 0.19.0", only: [:dev]}, # Code coverage - {:excoveralls, "~> 0.10.0", only: [:test], github: "KronicDeth/excoveralls", branch: "circle-workflows"}, - # Release - {:distillery, "~> 2.0", runtime: false} + {:excoveralls, "~> 0.10.0", only: [:test], github: "KronicDeth/excoveralls", branch: "circle-workflows"} ] end end diff --git a/mix.lock b/mix.lock index 68baad2790..6d76e8744e 100644 --- a/mix.lock +++ b/mix.lock @@ -24,6 +24,7 @@ "dataloader": {:hex, :dataloader, "1.0.4", "7c2345c53c9e5b61420013fc53c8463ba347a938b61f66677eb47d9c4a53ac5d", [:mix], [{:ecto, ">= 0.0.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"}, "db_connection": {:hex, :db_connection, "1.1.3", "89b30ca1ef0a3b469b1c779579590688561d586694a3ce8792985d4d7e575a61", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, optional: false]}, {:poolboy, "~> 1.5", [hex: :poolboy, optional: true]}, {:sbroker, "~> 1.0", [hex: :sbroker, optional: true]}]}, "decimal": {:hex, :decimal, "1.5.0", "b0433a36d0e2430e3d50291b1c65f53c37d56f83665b43d79963684865beab68", [:mix], []}, + "decorator": {:hex, :decorator, "1.2.4", "31dfff6143d37f0b68d0bffb3b9f18ace14fea54d4f1b5e4f86ead6f00d9ff6e", [:mix], [], "hexpm"}, "deep_merge": {:hex, :deep_merge, "0.2.0", "c1050fa2edf4848b9f556fba1b75afc66608a4219659e3311d9c9427b5b680b3", [:mix], [], "hexpm"}, "dialyxir": {:hex, :dialyxir, "0.5.1", "b331b091720fd93e878137add264bac4f644e1ddae07a70bf7062c7862c4b952", [:mix], []}, "distillery": {:hex, :distillery, "2.0.12", "6e78fe042df82610ac3fa50bd7d2d8190ad287d120d3cd1682d83a44e8b34dfb", [:mix], [{:artificery, "~> 0.2", [hex: :artificery, repo: "hexpm", optional: false]}], "hexpm"}, @@ -62,13 +63,15 @@ "math": {:hex, :math, "0.3.0", "e14e7291115201cb155a3567e66d196bf5088a6f55b030d598107d7ae934a11c", [:mix], []}, "meck": {:hex, :meck, "0.8.12", "1f7b1a9f5d12c511848fec26bbefd09a21e1432eadb8982d9a8aceb9891a3cf2", [:rebar3], [], "hexpm"}, "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], []}, - "mime": {:hex, :mime, "1.3.0", "5e8d45a39e95c650900d03f897fbf99ae04f60ab1daa4a34c7a20a5151b7a5fe", [:mix], [], "hexpm"}, + "mime": {:hex, :mime, "1.3.1", "30ce04ab3175b6ad0bdce0035cba77bba68b813d523d1aac73d9781b4d193cf8", [:mix], [], "hexpm"}, "mimerl": {:hex, :mimerl, "1.0.2", "993f9b0e084083405ed8252b99460c4f0563e41729ab42d9074fd5e52439be88", [:rebar3], []}, "mix_erlang_tasks": {:hex, :mix_erlang_tasks, "0.1.0", "36819fec60b80689eb1380938675af215565a89320a9e29c72c70d97512e4649", [:mix], [], "hexpm"}, "mochiweb": {:hex, :mochiweb, "2.18.0", "eb55f1db3e6e960fac4e6db4e2db9ec3602cc9f30b86cd1481d56545c3145d2e", [:rebar3], [], "hexpm"}, "mock": {:hex, :mock, "0.3.2", "e98e998fd76c191c7e1a9557c8617912c53df3d4a6132f561eb762b699ef59fa", [:mix], [{:meck, "~> 0.8.8", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm"}, "mox": {:hex, :mox, "0.4.0", "7f120840f7d626184a3d65de36189ca6f37d432e5d63acd80045198e4c5f7e6e", [:mix], [], "hexpm"}, + "msgpax": {:hex, :msgpax, "1.1.0", "e31625e256db2decca1ae2b841f21b4d2483b1332649ce3ebc96c7ff7a4986e3", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm"}, "nimble_parsec": {:hex, :nimble_parsec, "0.4.0", "ee261bb53214943679422be70f1658fff573c5d0b0a1ecd0f18738944f818efe", [:mix], [], "hexpm"}, + "optimal": {:hex, :optimal, "0.3.6", "46bbf52fbbbd238cda81e02560caa84f93a53c75620f1fe19e81e4ae7b07d1dd", [:mix], [], "hexpm"}, "parallel_stream": {:hex, :parallel_stream, "1.0.6", "b967be2b23f0f6787fab7ed681b4c45a215a81481fb62b01a5b750fa8f30f76c", [:mix], []}, "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm"}, "phoenix": {:hex, :phoenix, "1.3.4", "aaa1b55e5523083a877bcbe9886d9ee180bf2c8754905323493c2ac325903dc5", [:mix], [{:cowboy, "~> 1.0", [hex: :cowboy, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.0", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.3.3 or ~> 1.4", [hex: :plug, repo: "hexpm", optional: false]}, {:poison, "~> 2.2 or ~> 3.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm"}, @@ -92,6 +95,10 @@ "ranch": {:hex, :ranch, "1.3.2", "e4965a144dc9fbe70e5c077c65e73c57165416a901bd02ea899cfd95aa890986", [:rebar3], []}, "set_locale": {:git, "https://github.com/minifast/set_locale.git", "da9ae029642bc0fbd9212c2aaf86c0adca70c084", [branch: "master"]}, "sobelow": {:hex, :sobelow, "0.7.1", "01a52ea8a19be0aa41ce969746f057e90f3994f1607c771968359718bd0e6988", [:mix], [], "hexpm"}, + "spandex": {:git, "https://github.com/spandex-project/spandex.git", "92992b4aaf3d92e031c2417ff2e6c9e94d27fe36", [branch: "allow-setting-trace-key"]}, + "spandex_datadog": {:hex, :spandex_datadog, "0.3.1", "984d27ad1f45cfd243509692f0f63b900a23b79566c529a644c7f3a2b4120603", [:mix], [{:msgpax, "~> 1.1", [hex: :msgpax, repo: "hexpm", optional: false]}, {:spandex, "~> 2.3", [hex: :spandex, repo: "hexpm", optional: false]}], "hexpm"}, + "spandex_ecto": {:hex, :spandex_ecto, "0.4.0", "deaeaddc11a35f1c551206c53d09bb93a0da5808dbef751430e465c8c7de01d3", [:mix], [{:spandex, "~> 2.2", [hex: :spandex, repo: "hexpm", optional: false]}], "hexpm"}, + "spandex_phoenix": {:hex, :spandex_phoenix, "0.3.0", "48b0a426bbe4eea3e579bbea77d5eb5a8d4b83d33c95616f9ba64b3ce2faef6c", [:mix], [{:plug, "~> 1.3", [hex: :plug, repo: "hexpm", optional: false]}, {:spandex, "~> 2.2", [hex: :spandex, repo: "hexpm", optional: false]}], "hexpm"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"}, "timex": {:hex, :timex, "3.4.1", "e63fc1a37453035e534c3febfe9b6b9e18583ec7b37fd9c390efdef97397d70b", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "timex_ecto": {:hex, :timex_ecto, "3.3.0", "d5bdef09928e7a60f10a0baa47ce653f29b43d6fee87b30b236b216d0e36b98d", [:mix], [{:ecto, "~> 2.2", [hex: :ecto, repo: "hexpm", optional: false]}, {:timex, "~> 3.1", [hex: :timex, repo: "hexpm", optional: false]}], "hexpm"},