Land #384: Pluggable Transport and HTTP transport modules

pull/409/head
Luke Imhoff 6 years ago committed by GitHub
commit b583e45a1d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 109
      .circleci/config.yml
  2. 11
      apps/ethereum_jsonrpc/README.md
  3. 12
      apps/ethereum_jsonrpc/config/config.exs
  4. 5
      apps/ethereum_jsonrpc/config/geth.exs
  5. 9
      apps/ethereum_jsonrpc/config/parity.exs
  6. 235
      apps/ethereum_jsonrpc/lib/ethereum_jsonrpc.ex
  7. 18
      apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/geth.ex
  8. 171
      apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/http.ex
  9. 20
      apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/http/httpoison.ex
  10. 19
      apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/parity.ex
  11. 29
      apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/parity/trace.ex
  12. 4
      apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/parity/trace/action.ex
  13. 34
      apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/receipts.ex
  14. 79
      apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/transport.ex
  15. 10
      apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/variant.ex
  16. 2
      apps/ethereum_jsonrpc/mix.exs
  17. 178
      apps/ethereum_jsonrpc/test/etheream_jsonrpc_test.exs
  18. 23
      apps/ethereum_jsonrpc/test/ethereum_jsonrpc/geth_test.exs
  19. 121
      apps/ethereum_jsonrpc/test/ethereum_jsonrpc/http/mox_test.exs
  20. 213
      apps/ethereum_jsonrpc/test/ethereum_jsonrpc/parity_test.exs
  21. 144
      apps/ethereum_jsonrpc/test/ethereum_jsonrpc/receipts_test.exs
  22. 69
      apps/ethereum_jsonrpc/test/support/ethereum_jsonrpc/case.ex
  23. 32
      apps/ethereum_jsonrpc/test/support/ethereum_jsonrpc/http/case.ex
  24. 4
      apps/ethereum_jsonrpc/test/test_helper.exs
  25. 5
      apps/explorer/lib/explorer/chain.ex
  26. 8
      apps/indexer/config/config.exs
  27. 7
      apps/indexer/config/dev.exs
  28. 13
      apps/indexer/config/dev/geth.exs
  29. 17
      apps/indexer/config/dev/parity.exs
  30. 7
      apps/indexer/config/prod.exs
  31. 13
      apps/indexer/config/prod/geth.exs
  32. 17
      apps/indexer/config/prod/parity.exs
  33. 1
      apps/indexer/config/test.exs
  34. 20
      apps/indexer/lib/indexer/address_balance_fetcher.ex
  35. 6
      apps/indexer/lib/indexer/application.ex
  36. 21
      apps/indexer/lib/indexer/block_fetcher.ex
  37. 76
      apps/indexer/lib/indexer/buffered_task.ex
  38. 20
      apps/indexer/lib/indexer/internal_transaction_fetcher.ex
  39. 14
      apps/indexer/lib/indexer/pending_transaction_fetcher.ex
  40. 4
      apps/indexer/mix.exs
  41. 106
      apps/indexer/test/indexer/address_balance_fetcher_test.exs
  42. 525
      apps/indexer/test/indexer/block_fetcher_test.exs
  43. 21
      apps/indexer/test/indexer/buffered_task_test.exs
  44. 130
      apps/indexer/test/indexer/internal_transaction_fetcher_test.exs
  45. 59
      apps/indexer/test/indexer/pending_transaction_fetcher_test.exs
  46. 2
      apps/indexer/test/test_helper.exs

@ -296,7 +296,7 @@ jobs:
name: Scan explorer_web for vulnerabilities
command: mix sobelow --config
working_directory: "apps/explorer_web"
test_geth:
test_geth_http:
docker:
# Ensure .tool-versions matches
- image: circleci/elixir:1.6.5-node-browsers
@ -306,7 +306,10 @@ jobs:
PGPASSWORD: postgres
# match POSTGRES_USER for postgres image below
PGUSER: postgres
ETHEREUM_JSONRPC_VARIANT: geth
ETHEREUM_JSONRPC_VARIANT: "EthereumJSONRPC.Geth"
ETHEREUM_JSONRPC_TRANSPORT: "EthereumJSONRPC.HTTP"
ETHEREUM_JSONRPC_HTTP: "EthereumJSONRPC.HTTP.HTTPoison"
ETHEREUM_JSONRPC_HTTP_URL: "https://mainnet.infura.io/8lTvJTKmHPCHazkneJsY"
- image: circleci/postgres:10.3-alpine
environment:
# Match apps/explorer/config/test.exs config :explorerer, Explorer.Repo, database
@ -333,7 +336,7 @@ jobs:
- store_test_results:
path: _build/test/junit
test_parity:
test_geth_mox:
docker:
# Ensure .tool-versions matches
- image: circleci/elixir:1.6.5-node-browsers
@ -343,7 +346,8 @@ jobs:
PGPASSWORD: postgres
# match POSTGRES_USER for postgres image below
PGUSER: postgres
ETHEREUM_JSONRPC_VARIANT: parity
ETHEREUM_JSONRPC_VARIANT: "EthereumJSONRPC.Geth"
ETHEREUM_JSONRPC_TRANSPORT: "EthereumJSONRPC.Mox"
- image: circleci/postgres:10.3-alpine
environment:
# Match apps/explorer/config/test.exs config :explorerer, Explorer.Repo, database
@ -366,7 +370,86 @@ jobs:
name: Wait for DB
command: dockerize -wait tcp://localhost:5432 -timeout 1m
- run: mix coveralls.circle --parallel --umbrella
- run: mix coveralls.circle --exclude no_geth --parallel --umbrella
- store_test_results:
path: _build/test/junit
test_parity_http:
docker:
# Ensure .tool-versions matches
- image: circleci/elixir:1.6.5-node-browsers
environment:
MIX_ENV: test
# match POSTGRES_PASSWORD for postgres image below
PGPASSWORD: postgres
# match POSTGRES_USER for postgres image below
PGUSER: postgres
ETHEREUM_JSONRPC_VARIANT: "EthereumJSONRPC.Parity"
# enable on-chain tests against Sokol instead of `mox` tests run locally
ETHEREUM_JSONRPC_TRANSPORT: "EthereumJSONRPC.HTTP"
ETHEREUM_JSONRPC_HTTP: "EthereumJSONRPC.HTTP.HTTPoison"
ETHEREUM_JSONRPC_HTTP_URL: "https://sokol-trace.poa.network"
- image: circleci/postgres:10.3-alpine
environment:
# Match apps/explorer/config/test.exs config :explorer, Explorer.Repo, database
POSTGRES_DB: explorer_test
# match PGPASSWORD for elixir image above
POSTGRES_PASSWORD: postgres
# match PGUSER for elixir image above
POSTGRES_USER: postgres
working_directory: ~/app
steps:
- attach_workspace:
at: .
- run: mix local.hex --force
- run: mix local.rebar --force
- run:
name: Wait for DB
command: dockerize -wait tcp://localhost:5432 -timeout 1m
- run: mix coveralls.circle --exclude no_parity --parallel --umbrella
- store_test_results:
path: _build/test/junit
test_parity_mox:
docker:
# Ensure .tool-versions matches
- image: circleci/elixir:1.6.5-node-browsers
environment:
MIX_ENV: test
# match POSTGRES_PASSWORD for postgres image below
PGPASSWORD: postgres
# match POSTGRES_USER for postgres image below
PGUSER: postgres
ETHEREUM_JSONRPC_VARIANT: "EthereumJSONRPC.Parity"
ETHEREUM_JSONRPC_TRANSPORT: "EthereumJSONRPC.Mox"
- image: circleci/postgres:10.3-alpine
environment:
# Match apps/explorer/config/test.exs config :explorer, Explorer.Repo, database
POSTGRES_DB: explorer_test
# match PGPASSWORD for elixir image above
POSTGRES_PASSWORD: postgres
# match PGUSER for elixir image above
POSTGRES_USER: postgres
working_directory: ~/app
steps:
- attach_workspace:
at: .
- run: mix local.hex --force
- run: mix local.rebar --force
- run:
name: Wait for DB
command: dockerize -wait tcp://localhost:5432 -timeout 1m
- run: mix coveralls.circle --exclude no_parity --parallel --umbrella
- store_test_results:
path: _build/test/junit
@ -394,8 +477,10 @@ workflows:
- eslint
- jest
- sobelow
- test_parity
- test_geth
- test_parity_http
- test_parity_mox
- test_geth_http
- test_geth_mox
- dialyzer:
requires:
- build
@ -411,9 +496,15 @@ workflows:
- sobelow:
requires:
- build
- test_parity:
- test_parity_http:
requires:
- build
- test_parity_mox:
requires:
- build
- test_geth_http:
requires:
- build
- test_geth:
- test_geth_mox:
requires:
- build

@ -20,6 +20,17 @@ via `:trace_url`. The trace URL and is used for
tracing nodes. The `:http` option is passed directly to the HTTP
library (`HTTPoison`), which forwards the options down to `:hackney`.
## Testing
By default, [`mox`](https://github.com/plataformatec/mox) will be used to mock the `EthereumJSONRPC.Transport` and `EthereumJSONRPC.HTTP` behaviours. They mocked behaviours returns differ based on the `EthereumJSONRPC.Variant`.
| `EthereumJSONRPC.Variant` | `EthereumJSONRPC.Transport` | `EthereumJSONRPC.HTTP` | `url` | Command | Usage(s) |
|:--------------------------|:----------------------------|:---------------------------------|:--------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------|
| `EthereumJSONRPC.Parity` | `EthereumJSONRPC.Mox` | `EthereumJSONRPC.HTTP.Mox` | N/A | `mix test` | Local, `circleci/config.yml` `test_parity_mox` job |
| `EthereumJSONRPC.Parity` | `EthereumJSONRPC.HTTP` | `EthereumJSONRPC.HTTP.HTTPoison` | `https://trace-sokol.poa.network` | `ETHEREUM_JSONRPC_VARIANT=EthereumJSONRPC.Parity ETHEREUM_JSONRPC_TRANSPORT=EthereumJSONRPC.HTTP ETHEREUM_JSONRPC_HTTP=EthereumJSONRPC.HTTP.HTTPoison ETHEREUM_JSONRPC_HTTP_URL=https://sokol-trace.poa.network mix test --exclude no_parity` | `.circleci/config.yml` `test_parity_http` job |
| `EthereumJSONRPC.Geth` | `EthereumJSONRPC.Mox` | `EthereumJSONRPC.HTTP.Mox` | N/A | `ETHEREUM_JSONRPC_VARIANT=EthereumJSONRPC.Geth mix test --exclude no_geth` | `.circleci/config.yml` `test_geth_http` job |
| `EthereumJSONRPC.Geth` | `EthereumJSONRPC.HTTP` | `EthereumJSONRPC.HTTP.HTTPoison` | `https://mainnet.infura.io/8lTvJTKmHPCHazkneJsY` | `ETHEREUM_JSONRPC_VARIANT=EthereumJSONRPC.Geth ETHEREUM_JSONRPC_TRANSPORT=EthereumJSONRPC.HTTP ETHEREUM_JSONRPC_HTTP=EthereumJSONRPC.HTTP.HTTPoison ETHEREUM_JSONRPC_HTTP_URL=https://mainnet.infura.io/8lTvJTKmHPCHazkneJsY mix test --exclude no_geth` | `.circleci/config.yml` `test_geth_http` job |
## Installation
The OTP application `:ethereum_jsonrpc` can be used in other umbrella

@ -1,12 +0,0 @@
# This file is responsible for configuring your application
# and its dependencies with the aid of the Mix.Config module.
use Mix.Config
config :ethereum_jsonrpc,
http: [recv_timeout: 60_000, timeout: 60_000, hackney: [pool: :ethereum_jsonrpc]]
variant = System.get_env("ETHEREUM_JSONRPC_VARIANT") || "parity"
# Import variant specific config. This must remain at the bottom
# of this file so it overrides the configuration defined above.
import_config "#{variant}.exs"

@ -1,5 +0,0 @@
use Mix.Config
config :ethereum_jsonrpc,
url: "https://mainnet.infura.io/8lTvJTKmHPCHazkneJsY",
variant: EthereumJSONRPC.Geth

@ -1,9 +0,0 @@
use Mix.Config
config :ethereum_jsonrpc,
url: "https://sokol.poa.network",
method_to_url: [
eth_getBalance: "https://sokol-trace.poa.network",
trace_replayTransaction: "https://sokol-trace.poa.network"
],
variant: EthereumJSONRPC.Parity

@ -16,10 +16,8 @@ defmodule EthereumJSONRPC do
directly to the HTTP library (`HTTPoison`), which forwards the options down to `:hackney`.
"""
require Logger
alias Explorer.Chain.Block
alias EthereumJSONRPC.{Blocks, Receipts, Transactions}
alias EthereumJSONRPC.{Blocks, Receipts, Transactions, Transport, Variant}
@typedoc """
Truncated 20-byte [KECCAK-256](https://en.wikipedia.org/wiki/SHA-3) hash encoded as a hexadecimal number in a
@ -42,6 +40,18 @@ defmodule EthereumJSONRPC do
"""
@type hash :: String.t()
@typedoc """
Named arguments to `json_rpc/2`.
* `:transport` - the `t:EthereumJSONRPC.Transport.t/0` callback module
* `:transport_options` - options passed to `c:EthereumJSONRPC.Transport.json_rpc/2`
* `:variant` - the `t:EthereumJSONRPC.Variant.t/0` callback module
"""
@type json_rpc_named_arguments :: [
{:transport, Transport.t()} | {:transport_options, Transport.options()} | {:variant, Variant.t()}
]
@typedoc """
8 byte [KECCAK-256](https://en.wikipedia.org/wiki/SHA-3) hash of the proof-of-work.
"""
@ -73,47 +83,13 @@ defmodule EthereumJSONRPC do
"""
@type timestamp :: String.t()
@doc """
Fetches configuration for this module under `key`
Configuration can be set a compile time using `config`
config :ethereume_jsonrpc, key, value
Configuration can be set a runtime using `Application.put_env/3`
Application.put_env(:ethereume_jsonrpc, key, value)
"""
def config(key) do
Application.fetch_env!(:ethereum_jsonrpc, key)
end
@doc """
Fetches the configured url for the specific `method` or the fallback `url`
Configuration for a specific `method` can be set in `method_to_url` `config`
config :ethereum_jsonrpc,
method_to_url: [
eth_getBalance: "method_to_url"
]
The fallback 'url' MUST we set if not all methods have a url set.
config :ethereum_jsonrpc,
url:
"""
def method_to_url(method) when is_atom(method) do
:ethereum_jsonrpc
|> Application.get_env(:method_to_url, [])
|> Keyword.get_lazy(method, fn -> config(:url) end)
end
@doc """
Fetches balance for each address `hash` at the `block_number`
"""
@spec fetch_balances([%{required(:block_quantity) => quantity, required(:hash_data) => data()}]) ::
@spec fetch_balances(
[%{required(:block_quantity) => quantity, required(:hash_data) => data()}],
json_rpc_named_arguments
) ::
{:ok,
[
%{
@ -123,13 +99,14 @@ defmodule EthereumJSONRPC do
}
]}
| {:error, reason :: term}
def fetch_balances(params_list) when is_list(params_list) do
def fetch_balances(params_list, json_rpc_named_arguments)
when is_list(params_list) and is_list(json_rpc_named_arguments) do
id_to_params = id_to_params(params_list)
with {:ok, responses} <-
id_to_params
|> get_balance_requests()
|> json_rpc(method_to_url(:eth_getBalance)) do
|> json_rpc(json_rpc_named_arguments) do
get_balance_responses_to_addresses_params(responses, id_to_params)
end
end
@ -139,10 +116,10 @@ defmodule EthereumJSONRPC do
Transaction data is included for each block.
"""
def fetch_blocks_by_hash(block_hashes) do
def fetch_blocks_by_hash(block_hashes, json_rpc_named_arguments) do
block_hashes
|> get_block_by_hash_requests()
|> json_rpc(method_to_url(:eth_getBlockByHash))
|> json_rpc(json_rpc_named_arguments)
|> handle_get_blocks()
|> case do
{:ok, _next, results} -> {:ok, results}
@ -153,10 +130,10 @@ defmodule EthereumJSONRPC do
@doc """
Fetches blocks by block number range.
"""
def fetch_blocks_by_range(_first.._last = range) do
def fetch_blocks_by_range(_first.._last = range, json_rpc_named_arguments) do
range
|> get_block_by_number_requests()
|> json_rpc(method_to_url(:eth_getBlockByNumber))
|> json_rpc(json_rpc_named_arguments)
|> handle_get_blocks()
end
@ -170,33 +147,40 @@ defmodule EthereumJSONRPC do
* `{:error, reason}` - other JSONRPC error.
"""
@spec fetch_block_number_by_tag(tag()) :: {:ok, non_neg_integer()} | {:error, reason :: :invalid_tag | term()}
def fetch_block_number_by_tag(tag) when tag in ~w(earliest latest pending) do
@spec fetch_block_number_by_tag(tag(), json_rpc_named_arguments) ::
{:ok, non_neg_integer()} | {:error, reason :: :invalid_tag | term()}
def fetch_block_number_by_tag(tag, json_rpc_named_arguments) when tag in ~w(earliest latest pending) do
tag
|> get_block_by_tag_request()
|> json_rpc(method_to_url(:eth_getBlockByNumber))
|> json_rpc(json_rpc_named_arguments)
|> handle_get_block_by_tag()
end
@doc """
Fetches internal transactions from variant API.
"""
def fetch_internal_transactions(params_list) when is_list(params_list) do
config(:variant).fetch_internal_transactions(params_list)
def fetch_internal_transactions(params_list, json_rpc_named_arguments) when is_list(params_list) do
Keyword.fetch!(json_rpc_named_arguments, :variant).fetch_internal_transactions(
params_list,
json_rpc_named_arguments
)
end
@doc """
Fetches pending transactions from variant API.
"""
def fetch_pending_transactions do
config(:variant).fetch_pending_transactions()
def fetch_pending_transactions(json_rpc_named_arguments) do
Keyword.fetch!(json_rpc_named_arguments, :variant).fetch_pending_transactions(json_rpc_named_arguments)
end
@spec fetch_transaction_receipts([
@spec fetch_transaction_receipts(
[
%{required(:gas) => non_neg_integer(), required(:hash) => hash, optional(atom) => any}
]) :: {:ok, %{logs: list(), receipts: list()}} | {:error, reason :: term}
def fetch_transaction_receipts(transactions_params) when is_list(transactions_params) do
Receipts.fetch(transactions_params)
],
json_rpc_named_arguments
) :: {:ok, %{logs: list(), receipts: list()}} | {:error, reason :: term}
def fetch_transaction_receipts(transactions_params, json_rpc_named_arguments) when is_list(transactions_params) do
Receipts.fetch(transactions_params, json_rpc_named_arguments)
end
@doc """
@ -218,26 +202,15 @@ defmodule EthereumJSONRPC do
* Handled response
* `{:error, reason}` if POST failes
"""
def json_rpc(payload, url) when is_list(payload) do
chunked_json_rpc(url, [payload], config(:http), [])
end
@spec json_rpc(Transport.request(), json_rpc_named_arguments) ::
{:ok, Transport.result()} | {:error, reason :: term()}
@spec json_rpc(Transport.batch_request(), json_rpc_named_arguments) ::
{:ok, Transport.batch_response()} | {:error, reason :: term()}
def json_rpc(request, named_arguments) when (is_map(request) or is_list(request)) and is_list(named_arguments) do
transport = Keyword.fetch!(named_arguments, :transport)
transport_options = Keyword.fetch!(named_arguments, :transport_options)
def json_rpc(payload, url) do
json = encode_json(payload)
case post(url, json, config(:http)) do
{:ok, %HTTPoison.Response{body: body, status_code: status_code}} ->
with {:ok, json} <-
decode_json(
request: [url: url, body: json],
response: [status_code: status_code, body: body]
) do
handle_response(json, status_code)
end
{:error, %HTTPoison.Error{reason: reason}} ->
{:error, reason}
end
transport.json_rpc(request, transport_options)
end
@doc """
@ -267,14 +240,10 @@ defmodule EthereumJSONRPC do
@doc """
A request payload for a JSONRPC.
"""
@spec request(%{id: term, method: String.t(), params: list()}) :: %{String.t() => term}
def request(%{id: id, method: method, params: params}) do
%{
"id" => id,
"jsonrpc" => "2.0",
"method" => method,
"params" => params
}
@spec request(%{id: non_neg_integer(), method: String.t(), params: list()}) :: Transport.request()
def request(%{id: id, method: method, params: params} = map)
when is_integer(id) and is_binary(method) and is_list(params) do
Map.put(map, :jsonrpc, "2.0")
end
@doc """
@ -286,52 +255,6 @@ defmodule EthereumJSONRPC do
|> Timex.from_unix()
end
defp chunked_json_rpc(_url, [], _options, decoded_response_bodies) when is_list(decoded_response_bodies) do
list =
decoded_response_bodies
|> Enum.reverse()
|> List.flatten()
{:ok, list}
end
defp chunked_json_rpc(url, [batch | tail] = chunks, options, decoded_response_bodies)
when is_list(batch) and is_list(tail) and is_list(decoded_response_bodies) do
json = encode_json(batch)
case post(url, json, options) do
{:ok, %HTTPoison.Response{status_code: 413} = response} ->
rechunk_json_rpc(url, chunks, options, response, decoded_response_bodies)
{:ok, %HTTPoison.Response{body: body, status_code: status_code}} ->
with {:ok, decoded_body} <-
decode_json(request: [url: url, body: json], response: [status_code: status_code, body: body]) do
chunked_json_rpc(url, tail, options, [decoded_body | decoded_response_bodies])
end
{:error, %HTTPoison.Error{reason: reason}} ->
{:error, reason}
end
end
defp rechunk_json_rpc(url, [batch | tail], options, response, decoded_response_bodies) do
case length(batch) do
# it can't be made any smaller
1 ->
Logger.error(fn ->
"413 Request Entity Too Large returned from single request batch. Cannot shrink batch further."
end)
{:error, response}
batch_size ->
split_size = div(batch_size, 2)
{first_chunk, second_chunk} = Enum.split(batch, split_size)
new_chunks = [first_chunk, second_chunk | tail]
chunked_json_rpc(url, new_chunks, options, decoded_response_bodies)
end
end
defp get_balance_requests(id_to_params) when is_map(id_to_params) do
Enum.map(id_to_params, fn {id, %{block_quantity: block_quantity, hash_data: hash_data}} ->
get_balance_request(%{id: id, block_quantity: block_quantity, hash_data: hash_data})
@ -367,7 +290,7 @@ defmodule EthereumJSONRPC do
{status, Enum.reverse(reversed)}
end
defp get_balance_response_to_address_params(%{"id" => id, "result" => fetched_balance_quantity}, id_to_params)
defp get_balance_response_to_address_params(%{id: id, result: fetched_balance_quantity}, id_to_params)
when is_map(id_to_params) do
%{block_quantity: block_quantity, hash_data: hash_data} = Map.fetch!(id_to_params, id)
@ -379,11 +302,11 @@ defmodule EthereumJSONRPC do
}}
end
defp get_balance_response_to_address_params(%{"id" => id, "error" => error}, id_to_params)
defp get_balance_response_to_address_params(%{id: id, error: error}, id_to_params)
when is_map(id_to_params) do
%{block_quantity: block_quantity, hash_data: hash_data} = Map.fetch!(id_to_params, id)
annotated_error = Map.merge(error, %{"blockNumber" => block_quantity, "hash" => hash_data})
annotated_error = Map.put(error, :data, %{"blockNumber" => block_quantity, "hash" => hash_data})
{:error, annotated_error}
end
@ -414,7 +337,7 @@ defmodule EthereumJSONRPC do
defp get_block_by_tag_request(tag) do
# eth_getBlockByNumber accepts either a number OR a tag
get_block_by_number_request(%{id: 1, tag: tag, transactions: :hashes})
get_block_by_number_request(%{id: 0, tag: tag, transactions: :hashes})
end
defp get_block_by_number_params(options) do
@ -444,34 +367,11 @@ defmodule EthereumJSONRPC do
end
end
defp encode_json(data), do: Jason.encode_to_iodata!(data)
defp decode_json(named_arguments) when is_list(named_arguments) do
response = Keyword.fetch!(named_arguments, :response)
response_body = Keyword.fetch!(response, :body)
with {:error, _} <- Jason.decode(response_body) do
case Keyword.fetch!(response, :status_code) do
# CloudFlare protected server return HTML errors for 502, so the JSON decode will fail
502 ->
request_url =
named_arguments
|> Keyword.fetch!(:request)
|> Keyword.fetch!(:url)
{:error, {:bad_gateway, request_url}}
_ ->
raise EthereumJSONRPC.DecodeError, named_arguments
end
end
end
defp handle_get_blocks({:ok, results}) do
{blocks, next} =
Enum.reduce(results, {[], :more}, fn
%{"result" => nil}, {blocks, _} -> {blocks, :end_of_chain}
%{"result" => %{} = block}, {blocks, next} -> {[block | blocks], next}
%{result: nil}, {blocks, _} -> {blocks, :end_of_chain}
%{result: %{} = block}, {blocks, next} -> {[block | blocks], next}
end)
elixir_blocks = Blocks.to_elixir(blocks)
@ -494,19 +394,4 @@ defmodule EthereumJSONRPC do
defp handle_get_block_by_tag({:error, %{"code" => -32602}}), do: {:error, :invalid_tag}
defp handle_get_block_by_tag({:error, _} = error), do: error
defp handle_response(resp, 200) do
case resp do
%{"error" => error} -> {:error, error}
%{"result" => result} -> {:ok, result}
end
end
defp handle_response(resp, _status) do
{:error, resp}
end
defp post(url, json, options) do
HTTPoison.post(url, json, [{"Content-Type", "application/json"}], options)
end
end

@ -8,27 +8,17 @@ defmodule EthereumJSONRPC.Geth do
@doc """
Internal transaction fetching is not supported currently for Geth.
To signal to the caller that fetching is not supported, `:ignore` is returned
iex> EthereumJSONRPC.Geth.fetch_internal_transactions([
...> "0x2ec382949ba0b22443aa4cb38267b1fb5e68e188109ac11f7a82f67571a0adf3"
...> ])
:ignore
To signal to the caller that fetching is not supported, `:ignore` is returned.
"""
@impl EthereumJSONRPC.Variant
def fetch_internal_transactions(transaction_params) when is_list(transaction_params),
def fetch_internal_transactions(transaction_params, _json_rpc_named_arguments) when is_list(transaction_params),
do: :ignore
@doc """
Pending transaction fetching is not supported currently for Geth.
To signal to the caller that fetching is not supported, `:ignore` is returned
iex> EthereumJSONRPC.Geth.fetch_pending_transactions()
:ignore
To signal to the caller that fetching is not supported, `:ignore` is returned.
"""
@impl EthereumJSONRPC.Variant
def fetch_pending_transactions, do: :ignore
def fetch_pending_transactions(_json_rpc_named_arguments), do: :ignore
end

@ -0,0 +1,171 @@
defmodule EthereumJSONRPC.HTTP do
@moduledoc """
JSONRPC over HTTP
"""
alias EthereumJSONRPC.Transport
require Logger
@behaviour Transport
@callback json_rpc(url :: String.t(), json :: iodata(), options :: term()) ::
{:ok, %{body: body :: String.t(), status_code: status_code :: pos_integer()}}
| {:error, reason :: term}
@impl Transport
def json_rpc(%{method: method} = request, options) when is_map(request) do
json = encode_json(request)
http = Keyword.fetch!(options, :http)
url = url(options, method)
http_options = Keyword.fetch!(options, :http_options)
with {:ok, %{body: body, status_code: code}} <- http.json_rpc(url, json, http_options),
{:ok, json} <- decode_json(request: [url: url, body: json], response: [status_code: code, body: body]) do
handle_response(json, code)
end
end
def json_rpc(batch_request, options) when is_list(batch_request) do
chunked_json_rpc([batch_request], options, [])
end
defp chunked_json_rpc([], _options, decoded_response_bodies) when is_list(decoded_response_bodies) do
list =
decoded_response_bodies
|> Enum.reverse()
|> List.flatten()
|> Enum.map(&standardize_response/1)
{:ok, list}
end
# JSONRPC 2.0 standard says that an empty batch (`[]`) returns an empty response (`""`), but an empty response isn't
# valid JSON, so instead act like it returns an empty list (`[]`)
defp chunked_json_rpc([[] | tail], options, decoded_response_bodies) do
chunked_json_rpc(tail, options, decoded_response_bodies)
end
defp chunked_json_rpc([[%{method: method} | _] = batch | tail] = chunks, options, decoded_response_bodies)
when is_list(tail) and is_list(decoded_response_bodies) do
http = Keyword.fetch!(options, :http)
url = url(options, method)
http_options = Keyword.fetch!(options, :http_options)
json = encode_json(batch)
case http.json_rpc(url, json, http_options) do
{:ok, %{status_code: 413} = response} ->
rechunk_json_rpc(chunks, options, response, decoded_response_bodies)
{:ok, %{body: body, status_code: status_code}} ->
with {:ok, decoded_body} <-
decode_json(request: [url: url, body: json], response: [status_code: status_code, body: body]) do
chunked_json_rpc(tail, options, [decoded_body | decoded_response_bodies])
end
{:error, _} = error ->
error
end
end
defp rechunk_json_rpc([batch | tail], options, response, decoded_response_bodies) do
case length(batch) do
# it can't be made any smaller
1 ->
Logger.error(fn ->
"413 Request Entity Too Large returned from single request batch. Cannot shrink batch further."
end)
{:error, response}
batch_size ->
split_size = div(batch_size, 2)
{first_chunk, second_chunk} = Enum.split(batch, split_size)
new_chunks = [first_chunk, second_chunk | tail]
chunked_json_rpc(new_chunks, options, decoded_response_bodies)
end
end
defp encode_json(data), do: Jason.encode_to_iodata!(data)
defp decode_json(named_arguments) when is_list(named_arguments) do
response = Keyword.fetch!(named_arguments, :response)
response_body = Keyword.fetch!(response, :body)
with {:error, _} <- Jason.decode(response_body) do
case Keyword.fetch!(response, :status_code) do
# CloudFlare protected server return HTML errors for 502, so the JSON decode will fail
502 ->
request_url =
named_arguments
|> Keyword.fetch!(:request)
|> Keyword.fetch!(:url)
{:error, {:bad_gateway, request_url}}
_ ->
raise EthereumJSONRPC.DecodeError, named_arguments
end
end
end
defp handle_response(resp, 200) do
case resp do
%{"error" => error} -> {:error, standardize_error(error)}
%{"result" => result} -> {:ok, result}
end
end
defp handle_response(resp, _status) do
{:error, resp}
end
# restrict response to only those fields supported by the JSON-RPC 2.0 standard, which means that level of keys is
# validated, so we can indicate that with switch to atom keys.
defp standardize_response(%{"jsonrpc" => "2.0" = jsonrpc, "id" => id} = unstandardized) when is_integer(id) do
standardized = %{jsonrpc: jsonrpc, id: id}
case unstandardized do
%{"result" => _, "error" => _} ->
raise ArgumentError,
"result and error keys are mutually exclusive in JSONRPC 2.0 response objects, but got #{unstandardized}"
%{"result" => result} ->
Map.put(standardized, :result, result)
%{"error" => error} ->
Map.put(standardized, :error, standardize_error(error))
end
end
# restrict error to only those fields supported by the JSON-RPC 2.0 standard, which means that level of keys is
# validated, so we can indicate that with switch to atom keys.
defp standardize_error(%{"code" => code, "message" => message} = unstandardized)
when is_integer(code) and is_binary(message) do
standardized = %{code: code, message: message}
case Map.fetch(unstandardized, "data") do
{:ok, data} -> Map.put(standardized, :data, data)
:error -> standardized
end
end
defp url(options, method) when is_list(options) and is_binary(method) do
with {:ok, method_to_url} <- Keyword.fetch(options, :method_to_url),
{:ok, method_atom} <- to_existing_atom(method),
{:ok, url} <- Keyword.fetch(method_to_url, method_atom) do
url
else
_ -> Keyword.fetch!(options, :url)
end
end
defp to_existing_atom(string) do
{:ok, String.to_existing_atom(string)}
rescue
ArgumentError ->
:error
end
end

@ -0,0 +1,20 @@
defmodule EthereumJSONRPC.HTTP.HTTPoison do
@moduledoc """
Uses `HTTPoison` for `EthereumJSONRPC.HTTP`
"""
alias EthereumJSONRPC.HTTP
@behaviour HTTP
@impl HTTP
def json_rpc(url, json, options) when is_binary(url) and is_list(options) do
case HTTPoison.post(url, json, [{"Content-Type", "application/json"}], options) do
{:ok, %HTTPoison.Response{body: body, status_code: status_code}} ->
{:ok, %{body: body, status_code: status_code}}
{:error, %HTTPoison.Error{reason: reason}} ->
{:error, reason}
end
end
end

@ -3,7 +3,7 @@ defmodule EthereumJSONRPC.Parity do
Ethereum JSONRPC methods that are only supported by [Parity](https://wiki.parity.io/).
"""
import EthereumJSONRPC, only: [id_to_params: 1, method_to_url: 1, json_rpc: 2, request: 1]
import EthereumJSONRPC, only: [id_to_params: 1, json_rpc: 2, request: 1]
alias EthereumJSONRPC.Parity.Traces
alias EthereumJSONRPC.{Transaction, Transactions}
@ -14,13 +14,13 @@ defmodule EthereumJSONRPC.Parity do
Fetches the `t:Explorer.Chain.InternalTransaction.changeset/2` params from the Parity trace URL.
"""
@impl EthereumJSONRPC.Variant
def fetch_internal_transactions(transactions_params) when is_list(transactions_params) do
def fetch_internal_transactions(transactions_params, json_rpc_named_arguments) when is_list(transactions_params) do
id_to_params = id_to_params(transactions_params)
with {:ok, responses} <-
id_to_params
|> trace_replay_transaction_requests()
|> json_rpc(method_to_url(:trace_replayTransaction)) do
|> json_rpc(json_rpc_named_arguments) do
trace_replay_transaction_responses_to_internal_transactions_params(responses, id_to_params)
end
end
@ -32,12 +32,13 @@ defmodule EthereumJSONRPC.Parity do
on the transactions that each node has seen and how each node prioritizes collating transactions into the next block.
"""
@impl EthereumJSONRPC.Variant
@spec fetch_pending_transactions() :: {:ok, [Transaction.params()]} | {:error, reason :: term}
def fetch_pending_transactions do
@spec fetch_pending_transactions(EthereumJSONRPC.json_rpc_named_arguments()) ::
{:ok, [Transaction.params()]} | {:error, reason :: term}
def fetch_pending_transactions(json_rpc_named_arguments) do
with {:ok, transactions} <-
%{id: 1, method: "parity_pendingTransactions", params: []}
|> request()
|> json_rpc(method_to_url(:parity_pendingTransactions)) do
|> json_rpc(json_rpc_named_arguments) do
transactions_params =
transactions
|> Transactions.to_elixir()
@ -94,7 +95,7 @@ defmodule EthereumJSONRPC.Parity do
end
end
defp trace_replay_transaction_response_to_traces(%{"id" => id, "result" => %{"trace" => traces}}, id_to_params)
defp trace_replay_transaction_response_to_traces(%{id: id, result: %{"trace" => traces}}, id_to_params)
when is_list(traces) and is_map(id_to_params) do
%{block_number: block_number, hash_data: transaction_hash} = Map.fetch!(id_to_params, id)
@ -108,11 +109,11 @@ defmodule EthereumJSONRPC.Parity do
{:ok, annotated_traces}
end
defp trace_replay_transaction_response_to_traces(%{"id" => id, "error" => error}, id_to_params)
defp trace_replay_transaction_response_to_traces(%{id: id, error: error}, id_to_params)
when is_map(id_to_params) do
%{block_number: block_number, hash_data: transaction_hash} = Map.fetch!(id_to_params, id)
annotated_error = Map.merge(error, %{"blockNumber" => block_number, "transactionHash" => transaction_hash})
annotated_error = Map.put(error, :data, %{"blockNumber" => block_number, "transactionHash" => transaction_hash})
{:error, annotated_error}
end

@ -245,7 +245,11 @@ defmodule EthereumJSONRPC.Parity.Trace do
def elixir_to_params(%{"type" => "suicide" = type} = elixir) do
%{
"action" => %{"address" => from_address_hash, "balance" => value, "refundAddress" => to_address_hash},
"action" => %{
"address" => from_address_hash,
"balance" => value,
"refundAddress" => to_address_hash
},
"blockNumber" => block_number,
"index" => index,
"traceAddress" => trace_address,
@ -406,23 +410,28 @@ defmodule EthereumJSONRPC.Parity.Trace do
"""
def to_elixir(%{"blockNumber" => _, "index" => _, "transactionHash" => _} = trace) when is_map(trace) do
def to_elixir(%{"blockNumber" => _, "index" => _, "transactionHash" => _} = trace)
when is_map(trace) do
Enum.into(trace, %{}, &entry_to_elixir/1)
end
def to_elixir(_) do
raise ArgumentError, ~S|Caller must `Map.put/2` `"blockNumber"`, `"index"`, and `"transactionHash"` in trace|
raise ArgumentError,
~S|Caller must `Map.put/2` `"blockNumber"`, `"index"`, and `"transactionHash"` in trace|
end
# subtraces is an actual integer in JSON and not hex-encoded
# traceAddress is a list of actual integers, not a list of hex-encoded
defp entry_to_elixir({key, _} = entry) when key in ~w(subtraces traceAddress transactionHash type output), do: entry
defp entry_to_elixir({key, _} = entry)
when key in ~w(subtraces traceAddress transactionHash type output),
do: entry
defp entry_to_elixir({"action" = key, action}) do
{key, Action.to_elixir(action)}
end
defp entry_to_elixir({"blockNumber", block_number} = entry) when is_integer(block_number), do: entry
defp entry_to_elixir({"blockNumber", block_number} = entry) when is_integer(block_number),
do: entry
defp entry_to_elixir({"error", reason} = entry) when is_binary(reason), do: entry
@ -432,7 +441,9 @@ defmodule EthereumJSONRPC.Parity.Trace do
{key, Result.to_elixir(result)}
end
defp put_call_error_or_result(params, %{"result" => %{"gasUsed" => gas_used, "output" => output}}) do
defp put_call_error_or_result(params, %{
"result" => %{"gasUsed" => gas_used, "output" => output}
}) do
Map.merge(params, %{gas_used: gas_used, output: output})
end
@ -441,7 +452,11 @@ defmodule EthereumJSONRPC.Parity.Trace do
end
defp put_create_error_or_result(params, %{
"result" => %{"address" => created_contract_address_hash, "code" => code, "gasUsed" => gas_used}
"result" => %{
"address" => created_contract_address_hash,
"code" => code,
"gasUsed" => gas_used
}
}) do
Map.merge(params, %{
created_contract_code: code,

@ -44,7 +44,9 @@ defmodule EthereumJSONRPC.Parity.Trace.Action do
Enum.into(action, %{}, &entry_to_elixir/1)
end
defp entry_to_elixir({key, _} = entry) when key in ~w(address callType from init input refundAddress to), do: entry
defp entry_to_elixir({key, _} = entry)
when key in ~w(address callType from init input refundAddress to),
do: entry
defp entry_to_elixir({key, quantity}) when key in ~w(balance gas value) do
{key, quantity_to_integer(quantity)}

@ -5,7 +5,7 @@ defmodule EthereumJSONRPC.Receipts do
requests.
"""
import EthereumJSONRPC, only: [config: 1, json_rpc: 2]
import EthereumJSONRPC, only: [json_rpc: 2, request: 1]
alias EthereumJSONRPC.{Logs, Receipt}
@ -111,14 +111,17 @@ defmodule EthereumJSONRPC.Receipts do
Enum.map(elixir, &Receipt.elixir_to_params/1)
end
@spec fetch([
@spec fetch(
[
%{
required(:gas) => non_neg_integer(),
required(:hash) => EthereumJSONRPC.hash(),
optional(atom) => any
}
]) :: {:ok, %{logs: list(), receipts: list()}} | {:error, reason :: term}
def fetch(transactions_params) when is_list(transactions_params) do
],
EthereumJSONRPC.json_rpc_named_arguments()
) :: {:ok, %{logs: list(), receipts: list()}} | {:error, reason :: term()}
def fetch(transactions_params, json_rpc_named_arguments) when is_list(transactions_params) do
{requests, id_to_transaction_params} =
transactions_params
|> Stream.with_index()
@ -129,10 +132,7 @@ defmodule EthereumJSONRPC.Receipts do
{requests, id_to_transaction_params}
end)
requests
|> json_rpc(config(:url))
|> case do
{:ok, responses} ->
with {:ok, responses} <- json_rpc(requests, json_rpc_named_arguments) do
elixir_receipts =
responses
|> responses_to_receipts(id_to_transaction_params)
@ -143,9 +143,6 @@ defmodule EthereumJSONRPC.Receipts do
logs = Logs.elixir_to_params(elixir_logs)
{:ok, %{logs: logs, receipts: receipts}}
{:error, _reason} = err ->
err
end
end
@ -216,17 +213,16 @@ defmodule EthereumJSONRPC.Receipts do
end
defp request(id, transaction_hash) when is_integer(id) and is_binary(transaction_hash) do
%{
"id" => id,
"jsonrpc" => "2.0",
"method" => "eth_getTransactionReceipt",
"params" => [transaction_hash]
}
request(%{
id: id,
method: "eth_getTransactionReceipt",
params: [transaction_hash]
})
end
defp response_to_receipt(%{"result" => nil}, _), do: %{}
defp response_to_receipt(%{result: nil}, _), do: %{}
defp response_to_receipt(%{"id" => id, "result" => receipt}, id_to_transaction_params) do
defp response_to_receipt(%{id: id, result: receipt}, id_to_transaction_params) do
gas =
id_to_transaction_params
|> Map.fetch!(id)

@ -0,0 +1,79 @@
defmodule EthereumJSONRPC.Transport do
@moduledoc """
The transport over which JSONRPC calls occur.
Various clients support the transports below:
* HTTP
* IPC
* WS
"""
@typedoc @moduledoc
@type t :: module
@typedoc """
The name of the JSONRPC method
"""
@type method :: String.t()
@typedoc """
[JSONRPC request object](https://www.jsonrpc.org/specification#request_object)
* `:jsonrpc` - a `t:String.t/0` specifying the JSONR-RPC protocol version. MUST be exactly `"2.0"`
* `:method` - a `t:String.t/0` containing the name of the method to be invoked.
* `:params` - a `t:list/0` for the positional parameters for the `"method"`.
* `:id` - a `non_neg_integer` that is unique for in a `t:batch_request/0`.
"""
@type request :: %{jsonrpc: String.t(), method: method, params: list(), id: non_neg_integer()}
@typedoc """
A batch of `t:request/0`. Each `t:request/0` in the batch must have a unique `"id"`.
"""
@type batch_request :: [request]
@type result :: term()
@typedoc """
[JSONRPC response object](https://www.jsonrpc.org/specification#response_object)
## Result
* `:jsonrpc` - a `t:String.t/0` specifying the JSONR-RPC protocol version. MUST be exactly `"2.0"`
* `:result` - the successful result of the request
* `:id` - the `"id'` of the `t:request/0` that correlates with this response
## Error
* `:jsonrpc` - a `t:String.t/0` specifying the JSONR-RPC protocol version. MUST be exactly `"2.0"`
* `:error:` - the `t:error/0`
* `:id` - the `"id'` of the `t:request/0` that correlates with this response
"""
@type response ::
%{jsonrpc: String.t(), result: result, id: non_neg_integer()}
| %{jsonrpc: String.t(), error: error, id: non_neg_integer()}
@typedoc """
[JSONRPC error object](https://www.jsonrpc.org/specification#error_object)
* `:code` - an `t:integer/0` indicating the error type
* `:message` -
"""
@type error :: %{required(:code) => integer(), required(:message) => String.t(), optional(:data) => term()}
@typedoc """
A batch of `t:response/0`. Each `t:response/0` will have an `"id"` corresponding to the `"id"` in the `t:request/0`.
"""
@type batch_response :: [response]
@typedoc """
Transport-specific options
"""
@type options :: term()
@callback json_rpc(request, options) :: {:ok, result} | {:error, reason :: term()}
@callback json_rpc(batch_request, options) :: {:ok, batch_response} | {:error, reason :: term()}
end

@ -6,6 +6,11 @@ defmodule EthereumJSONRPC.Variant do
alias EthereumJSONRPC.Transaction
@typedoc """
A module that implements the `EthereumJSONRPC.Variant` behaviour callbacks.
"""
@type t :: module
@type internal_transaction_params :: map()
@doc """
@ -18,7 +23,7 @@ defmodule EthereumJSONRPC.Variant do
internal transactions
* `:ignore` - the variant does not support fetching internal transactions.
"""
@callback fetch_internal_transactions([Transaction.params()]) ::
@callback fetch_internal_transactions([Transaction.params()], EthereumJSONRPC.json_rpc_named_arguments()) ::
{:ok, [internal_transaction_params]} | {:error, reason :: term} | :ignore
@doc """
@ -31,5 +36,6 @@ defmodule EthereumJSONRPC.Variant do
* `{:error, reason}` - there was one or more errors with `reason` in fetching the pending transactions
* `:ignore` - the variant does not support fetching pending transactions.
"""
@callback fetch_pending_transactions() :: {:ok, [Transaction.params()]} | {:error, reason :: term} | :ignore
@callback fetch_pending_transactions(EthereumJSONRPC.json_rpc_named_arguments()) ::
{:ok, [Transaction.params()]} | {:error, reason :: term} | :ignore
end

@ -69,6 +69,8 @@ defmodule EthereumJsonrpc.MixProject do
{:httpoison, "~> 1.0", override: true},
# Decode/Encode JSON for JSONRPC
{:jason, "~> 1.0"},
# Mocking `EthereumJSONRPC.Transport` and `EthereumJSONRPC.HTTP` so we avoid hitting real chains for local testing
{:mox, "~> 0.3.2", only: [:test]},
# Convert unix timestamps in JSONRPC to DateTimes
{:timex, "~> 3.1.24"}
]

@ -1,71 +1,133 @@
defmodule EthereumJSONRPCTest do
use ExUnit.Case, async: true
use EthereumJSONRPC.Case, async: true
import EthereumJSONRPC.Case
import Mox
setup :verify_on_exit!
@moduletag :capture_log
setup do
%{variant: EthereumJSONRPC.config(:variant)}
describe "fetch_balances/1" do
test "with all valid hash_data returns {:ok, addresses_params}", %{
json_rpc_named_arguments: json_rpc_named_arguments
} do
expected_fetched_balance =
case Keyword.fetch!(json_rpc_named_arguments, :variant) do
EthereumJSONRPC.Geth -> 0
EthereumJSONRPC.Parity -> 1
variant -> raise ArgumentError, "Unsupported variant (#{variant}})"
end
describe "fetch_balances/1" do
test "with all valid hash_data returns {:ok, addresses_params}", %{variant: variant} do
assert {:ok,
if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do
expect(EthereumJSONRPC.Mox, :json_rpc, fn _json, _options ->
{:ok, [%{id: 0, result: EthereumJSONRPC.integer_to_quantity(expected_fetched_balance)}]}
end)
end
hash = "0x8bf38d4764929064f2d4d3a56520a76ab3df415b"
assert EthereumJSONRPC.fetch_balances(
[
%{block_quantity: "0x1", hash_data: hash}
],
json_rpc_named_arguments
) ==
{:ok,
[
%{
fetched_balance: fetched_balance,
fetched_balance: expected_fetched_balance,
fetched_balance_block_number: 1,
hash: "0x8bf38d4764929064f2d4d3a56520a76ab3df415b"
hash: hash
}
]} =
EthereumJSONRPC.fetch_balances([
%{block_quantity: "0x1", hash_data: "0x8bf38d4764929064f2d4d3a56520a76ab3df415b"}
])
case variant do
EthereumJSONRPC.Geth ->
assert fetched_balance == 0
EthereumJSONRPC.Parity ->
assert fetched_balance == 1
_ ->
raise ArgumentError, "Unsupported variant (#{variant}})"
end
]}
end
test "with all invalid hash_data returns {:error, reasons}", %{variant: variant} do
assert {:error, reasons} = EthereumJSONRPC.fetch_balances([%{block_quantity: "0x1", hash_data: "0x0"}])
assert is_list(reasons)
assert length(reasons) == 1
[reason] = reasons
assert %{
"blockNumber" => "0x1",
"code" => -32602,
"hash" => "0x0",
"message" => message
} = reason
test "with all invalid hash_data returns {:error, reasons}", %{json_rpc_named_arguments: json_rpc_named_arguments} do
variant = Keyword.fetch!(json_rpc_named_arguments, :variant)
expected_message =
case variant do
EthereumJSONRPC.Geth ->
assert message ==
"invalid argument 0: json: cannot unmarshal hex string of odd length into Go value of type common.Address"
EthereumJSONRPC.Parity ->
assert message ==
"Invalid params: invalid length 1, expected a 0x-prefixed, padded, hex-encoded hash with length 40."
_ ->
raise ArgumentError, "Unsupported variant (#{variant}})"
end
if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do
expect(EthereumJSONRPC.Mox, :json_rpc, fn _json, _options ->
{:ok,
[
%{
id: 0,
error: %{
code: -32602,
message: expected_message
}
}
]}
end)
end
assert {:error,
[
%{
code: -32602,
data: %{"blockNumber" => "0x1", "hash" => "0x0"},
message: ^expected_message
}
]} =
EthereumJSONRPC.fetch_balances([%{block_quantity: "0x1", hash_data: "0x0"}], json_rpc_named_arguments)
end
test "with a mix of valid and invalid hash_data returns {:error, reasons}", %{
json_rpc_named_arguments: json_rpc_named_arguments
} do
if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do
expect(EthereumJSONRPC.Mox, :json_rpc, fn _json, _options ->
{
:ok,
[
%{
id: 0,
result: "0x0"
},
%{
id: 1,
result: "0x1"
},
%{
id: 2,
error: %{
code: -32602,
message:
"Invalid params: invalid length 1, expected a 0x-prefixed, padded, hex-encoded hash with length 40."
}
},
%{
id: 3,
result: "0x3"
},
%{
id: 4,
error: %{
code: -32602,
message:
"Invalid params: invalid length 1, expected a 0x-prefixed, padded, hex-encoded hash with length 40."
}
}
]
}
end)
end
test "with a mix of valid and invalid hash_data returns {:error, reasons}" do
assert {:error, reasons} =
EthereumJSONRPC.fetch_balances([
EthereumJSONRPC.fetch_balances(
[
# start with :ok
%{
block_quantity: "0x1",
@ -91,18 +153,26 @@ defmodule EthereumJSONRPCTest do
block_quantity: "0x4",
hash_data: "0x5"
}
])
],
json_rpc_named_arguments
)
assert is_list(reasons)
assert length(reasons) > 1
end
end
describe "fetch_block_number_by_tag/1" do
describe "fetch_block_number_by_tag" do
@tag capture_log: false
test "with earliest" do
test "with earliest", %{json_rpc_named_arguments: json_rpc_named_arguments} do
if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do
expect(EthereumJSONRPC.Mox, :json_rpc, fn _json, _options ->
{:ok, %{"number" => "0x0"}}
end)
end
log_bad_gateway(
fn -> EthereumJSONRPC.fetch_block_number_by_tag("earliest") end,
fn -> EthereumJSONRPC.fetch_block_number_by_tag("earliest", json_rpc_named_arguments) end,
fn result ->
assert {:ok, 0} = result
end
@ -110,9 +180,15 @@ defmodule EthereumJSONRPCTest do
end
@tag capture_log: false
test "with latest" do
test "with latest", %{json_rpc_named_arguments: json_rpc_named_arguments} do
if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do
expect(EthereumJSONRPC.Mox, :json_rpc, fn _json, _options ->
{:ok, %{"number" => "0x1"}}
end)
end
log_bad_gateway(
fn -> EthereumJSONRPC.fetch_block_number_by_tag("latest") end,
fn -> EthereumJSONRPC.fetch_block_number_by_tag("latest", json_rpc_named_arguments) end,
fn result ->
assert {:ok, number} = result
assert number > 0
@ -121,9 +197,15 @@ defmodule EthereumJSONRPCTest do
end
@tag capture_log: false
test "with pending" do
test "with pending", %{json_rpc_named_arguments: json_rpc_named_arguments} do
if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do
expect(EthereumJSONRPC.Mox, :json_rpc, fn _json, _options ->
{:ok, %{"number" => "0x2"}}
end)
end
log_bad_gateway(
fn -> EthereumJSONRPC.fetch_block_number_by_tag("pending") end,
fn -> EthereumJSONRPC.fetch_block_number_by_tag("pending", json_rpc_named_arguments) end,
fn result ->
assert {:ok, number} = result
assert number > 0

@ -1,5 +1,24 @@
defmodule EthereumJSONRPC.GethTest do
use ExUnit.Case, async: false
use EthereumJSONRPC.Case, async: false
doctest EthereumJSONRPC.Geth
alias EthereumJSONRPC.Geth
@moduletag :no_parity
describe "fetch_internal_transactions/2" do
test "is not supported", %{json_rpc_named_arguments: json_rpc_named_arguments} do
Geth.fetch_internal_transactions(
[
"0x2ec382949ba0b22443aa4cb38267b1fb5e68e188109ac11f7a82f67571a0adf3"
],
json_rpc_named_arguments
)
end
end
describe "fetch_pending_transactions/1" do
test "is not supported", %{json_rpc_named_arguments: json_rpc_named_arguments} do
EthereumJSONRPC.Geth.fetch_pending_transactions(json_rpc_named_arguments)
end
end
end

@ -0,0 +1,121 @@
defmodule EthereumJSONRPC.HTTP.MoxTest do
@moduledoc """
Tests differences in behavior of `EthereumJSONRPC` when `EthereumJSONRPC.HTTP` is used as the transport that are too
detrimental to run against Sokol, so uses `EthereumJSONRPC.HTTP.Mox` instead.
"""
use ExUnit.Case, async: true
import EthereumJSONRPC, only: [request: 1]
import EthereumJSONRPC.HTTP.Case
import Mox
setup do
%{
json_rpc_named_arguments: [
transport: EthereumJSONRPC.HTTP,
transport_options: [
http: EthereumJSONRPC.HTTP.Mox,
url: url(),
http_options: http_options()
]
]
}
end
setup :verify_on_exit!
describe "json_rpc/2" do
# regression test for https://github.com/poanetwork/poa-explorer/issues/254
#
# this test triggered a DoS with CloudFlare reporting 502 Bad Gateway
# (see https://github.com/poanetwork/poa-explorer/issues/340), so it can't be run against the real Sokol chain and
# must use `mox` to fake it.
test "transparently splits batch payloads that would trigger a 413 Request Entity Too Large", %{
json_rpc_named_arguments: json_rpc_named_arguments
} do
if json_rpc_named_arguments[:transport_options][:http] == EthereumJSONRPC.HTTP.Mox do
EthereumJSONRPC.HTTP.Mox
|> expect(:json_rpc, 2, fn _url, json, _options ->
assert IO.iodata_to_binary(json) =~ ":13000"
{:ok, %{body: "413 Request Entity Too Large", status_code: 413}}
end)
|> expect(:json_rpc, fn _url, json, _options ->
json_binary = IO.iodata_to_binary(json)
refute json_binary =~ ":13000"
assert json_binary =~ ":6499"
body =
0..6499
|> Enum.map(fn id ->
%{jsonrpc: "2.0", id: id, result: %{number: EthereumJSONRPC.integer_to_quantity(id)}}
end)
|> Jason.encode!()
{:ok, %{body: body, status_code: 200}}
end)
|> expect(:json_rpc, fn _url, json, _optons ->
json_binary = IO.iodata_to_binary(json)
refute json_binary =~ ":6499"
assert json_binary =~ ":6500"
assert json_binary =~ ":13000"
body =
6500..13000
|> Enum.map(fn id ->
%{jsonrpc: "2.0", id: id, result: %{number: EthereumJSONRPC.integer_to_quantity(id)}}
end)
|> Jason.encode!()
{:ok, %{body: body, status_code: 200}}
end)
end
block_numbers = 0..13000
payload =
block_numbers
|> Stream.with_index()
|> Enum.map(&get_block_by_number_request/1)
assert_payload_too_large(payload, json_rpc_named_arguments)
assert {:ok, responses} = EthereumJSONRPC.json_rpc(payload, json_rpc_named_arguments)
assert Enum.count(responses) == Enum.count(block_numbers)
block_number_set = MapSet.new(block_numbers)
response_block_number_set =
Enum.into(responses, MapSet.new(), fn %{result: %{"number" => quantity}} ->
EthereumJSONRPC.quantity_to_integer(quantity)
end)
assert MapSet.equal?(response_block_number_set, block_number_set)
end
end
defp assert_payload_too_large(payload, json_rpc_named_arguments) do
assert Keyword.fetch!(json_rpc_named_arguments, :transport) == EthereumJSONRPC.HTTP
transport_options = Keyword.fetch!(json_rpc_named_arguments, :transport_options)
http = Keyword.fetch!(transport_options, :http)
url = Keyword.fetch!(transport_options, :url)
json = Jason.encode_to_iodata!(payload)
http_options = Keyword.fetch!(transport_options, :http_options)
assert {:ok, %{body: body, status_code: 413}} = http.json_rpc(url, json, http_options)
assert body =~ "413 Request Entity Too Large"
end
defp get_block_by_number_request({block_number, id}) do
request(%{
id: id,
method: "eth_getBlockByNumber",
params: [EthereumJSONRPC.integer_to_quantity(block_number), true]
})
end
end

@ -1,61 +1,181 @@
defmodule EthereumJSONRPC.ParityTest do
use ExUnit.Case, async: true
use EthereumJSONRPC.Case
import EthereumJSONRPC, only: [integer_to_quantity: 1]
import Mox
setup :verify_on_exit!
doctest EthereumJSONRPC.Parity
@moduletag :no_geth
describe "fetch_internal_transactions/1" do
@tag :no_geth
test "with all valid transaction_params returns {:ok, transactions_params}" do
assert EthereumJSONRPC.Parity.fetch_internal_transactions([
test "with all valid transaction_params returns {:ok, transactions_params}", %{
json_rpc_named_arguments: json_rpc_named_arguments
} do
from_address_hash = "0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca"
gas = 4_533_872
init =
"0x6060604052341561000f57600080fd5b60405160208061071a83398101604052808051906020019091905050806000806101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff1602179055506003600160006001600281111561007e57fe5b60ff1660ff168152602001908152602001600020819055506002600160006002808111156100a857fe5b60ff1660ff168152602001908152602001600020819055505061064a806100d06000396000f30060606040526004361061008e576000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff168063247b3210146100935780632ffdfc8a146100bc57806374294144146100f6578063ae4b1b5b14610125578063bf7370d11461017a578063d1104cb2146101a3578063eecd1079146101f8578063fcff021c14610221575b600080fd5b341561009e57600080fd5b6100a661024a565b6040518082815260200191505060405180910390f35b34156100c757600080fd5b6100e0600480803560ff16906020019091905050610253565b6040518082815260200191505060405180910390f35b341561010157600080fd5b610123600480803590602001909190803560ff16906020019091905050610276565b005b341561013057600080fd5b61013861037a565b604051808273ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16815260200191505060405180910390f35b341561018557600080fd5b61018d61039f565b6040518082815260200191505060405180910390f35b34156101ae57600080fd5b6101b66104d9565b604051808273ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16815260200191505060405180910390f35b341561020357600080fd5b61020b610588565b6040518082815260200191505060405180910390f35b341561022c57600080fd5b6102346105bd565b6040518082815260200191505060405180910390f35b600060c8905090565b6000600160008360ff1660ff168152602001908152602001600020549050919050565b61027e6104d9565b73ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff161415156102b757600080fd5b60008160ff161115156102c957600080fd5b6002808111156102d557fe5b60ff168160ff16111515156102e957600080fd5b6000821180156103125750600160008260ff1660ff168152602001908152602001600020548214155b151561031d57600080fd5b81600160008360ff1660ff168152602001908152602001600020819055508060ff167fe868bbbdd6cd2efcd9ba6e0129d43c349b0645524aba13f8a43bfc7c5ffb0889836040518082815260200191505060405180910390a25050565b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1681565b6000806000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16638b8414c46000604051602001526040518163ffffffff167c0100000000000000000000000000000000000000000000000000000000028152600401602060405180830381600087803b151561042f57600080fd5b6102c65a03f1151561044057600080fd5b5050506040518051905090508073ffffffffffffffffffffffffffffffffffffffff16630eaba26a6000604051602001526040518163ffffffff167c0100000000000000000000000000000000000000000000000000000000028152600401602060405180830381600087803b15156104b857600080fd5b6102c65a03f115156104c957600080fd5b5050506040518051905091505090565b60008060009054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff1663a3b3fff16000604051602001526040518163ffffffff167c0100000000000000000000000000000000000000000000000000000000028152600401602060405180830381600087803b151561056857600080fd5b6102c65a03f1151561057957600080fd5b50505060405180519050905090565b60006105b860016105aa600261059c61039f565b6105e590919063ffffffff16565b61060090919063ffffffff16565b905090565b60006105e06105ca61039f565b6105d261024a565b6105e590919063ffffffff16565b905090565b60008082848115156105f357fe5b0490508091505092915050565b600080828401905083811015151561061457fe5b80915050929150505600a165627a7a723058206b7eef2a57eb659d5e77e45ab5bc074e99c6a841921038cdb931e119c6aac46c0029000000000000000000000000862d67cb0773ee3f8ce7ea89b328ffea861ab3ef"
value = 0
block_number = 1
index = 0
created_contract_address_hash = "0x1e0eaa06d02f965be2dfe0bc9ff52b2d82133461"
created_contract_code =
"0x60606040526004361061008e576000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff168063247b3210146100935780632ffdfc8a146100bc57806374294144146100f6578063ae4b1b5b14610125578063bf7370d11461017a578063d1104cb2146101a3578063eecd1079146101f8578063fcff021c14610221575b600080fd5b341561009e57600080fd5b6100a661024a565b6040518082815260200191505060405180910390f35b34156100c757600080fd5b6100e0600480803560ff16906020019091905050610253565b6040518082815260200191505060405180910390f35b341561010157600080fd5b610123600480803590602001909190803560ff16906020019091905050610276565b005b341561013057600080fd5b61013861037a565b604051808273ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16815260200191505060405180910390f35b341561018557600080fd5b61018d61039f565b6040518082815260200191505060405180910390f35b34156101ae57600080fd5b6101b66104d9565b604051808273ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16815260200191505060405180910390f35b341561020357600080fd5b61020b610588565b6040518082815260200191505060405180910390f35b341561022c57600080fd5b6102346105bd565b6040518082815260200191505060405180910390f35b600060c8905090565b6000600160008360ff1660ff168152602001908152602001600020549050919050565b61027e6104d9565b73ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff161415156102b757600080fd5b60008160ff161115156102c957600080fd5b6002808111156102d557fe5b60ff168160ff16111515156102e957600080fd5b6000821180156103125750600160008260ff1660ff168152602001908152602001600020548214155b151561031d57600080fd5b81600160008360ff1660ff168152602001908152602001600020819055508060ff167fe868bbbdd6cd2efcd9ba6e0129d43c349b0645524aba13f8a43bfc7c5ffb0889836040518082815260200191505060405180910390a25050565b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1681565b6000806000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16638b8414c46000604051602001526040518163ffffffff167c0100000000000000000000000000000000000000000000000000000000028152600401602060405180830381600087803b151561042f57600080fd5b6102c65a03f1151561044057600080fd5b5050506040518051905090508073ffffffffffffffffffffffffffffffffffffffff16630eaba26a6000604051602001526040518163ffffffff167c0100000000000000000000000000000000000000000000000000000000028152600401602060405180830381600087803b15156104b857600080fd5b6102c65a03f115156104c957600080fd5b5050506040518051905091505090565b60008060009054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff1663a3b3fff16000604051602001526040518163ffffffff167c0100000000000000000000000000000000000000000000000000000000028152600401602060405180830381600087803b151561056857600080fd5b6102c65a03f1151561057957600080fd5b50505060405180519050905090565b60006105b860016105aa600261059c61039f565b6105e590919063ffffffff16565b61060090919063ffffffff16565b905090565b60006105e06105ca61039f565b6105d261024a565b6105e590919063ffffffff16565b905090565b60008082848115156105f357fe5b0490508091505092915050565b600080828401905083811015151561061457fe5b80915050929150505600a165627a7a723058206b7eef2a57eb659d5e77e45ab5bc074e99c6a841921038cdb931e119c6aac46c0029"
gas_used = 382_953
trace_address = []
transaction_hash = "0x0fa6f723216dba694337f9bb37d8870725655bdf2573526a39454685659e39b1"
type = "create"
if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do
expect(EthereumJSONRPC.Mox, :json_rpc, fn _json, _options ->
{:ok,
[
%{
block_number: 1,
hash_data: "0x0fa6f723216dba694337f9bb37d8870725655bdf2573526a39454685659e39b1"
id: 0,
result: %{
"trace" => [
%{
"action" => %{
"from" => from_address_hash,
"gas" => integer_to_quantity(gas),
"init" => init,
"value" => integer_to_quantity(value)
},
"blockNumber" => block_number,
"index" => index,
"result" => %{
"address" => created_contract_address_hash,
"code" => created_contract_code,
"gasUsed" => integer_to_quantity(gas_used)
},
"traceAddress" => trace_address,
"transactionHash" => transaction_hash,
"type" => type
}
]
}
}
]) == {
]}
end)
end
assert EthereumJSONRPC.Parity.fetch_internal_transactions(
[
%{
block_number: block_number,
hash_data: transaction_hash
}
],
json_rpc_named_arguments
) == {
:ok,
[
%{
block_number: 1,
created_contract_address_hash: "0x1e0eaa06d02f965be2dfe0bc9ff52b2d82133461",
created_contract_code:
"0x60606040526004361061008e576000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff168063247b3210146100935780632ffdfc8a146100bc57806374294144146100f6578063ae4b1b5b14610125578063bf7370d11461017a578063d1104cb2146101a3578063eecd1079146101f8578063fcff021c14610221575b600080fd5b341561009e57600080fd5b6100a661024a565b6040518082815260200191505060405180910390f35b34156100c757600080fd5b6100e0600480803560ff16906020019091905050610253565b6040518082815260200191505060405180910390f35b341561010157600080fd5b610123600480803590602001909190803560ff16906020019091905050610276565b005b341561013057600080fd5b61013861037a565b604051808273ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16815260200191505060405180910390f35b341561018557600080fd5b61018d61039f565b6040518082815260200191505060405180910390f35b34156101ae57600080fd5b6101b66104d9565b604051808273ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16815260200191505060405180910390f35b341561020357600080fd5b61020b610588565b6040518082815260200191505060405180910390f35b341561022c57600080fd5b6102346105bd565b6040518082815260200191505060405180910390f35b600060c8905090565b6000600160008360ff1660ff168152602001908152602001600020549050919050565b61027e6104d9565b73ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff161415156102b757600080fd5b60008160ff161115156102c957600080fd5b6002808111156102d557fe5b60ff168160ff16111515156102e957600080fd5b6000821180156103125750600160008260ff1660ff168152602001908152602001600020548214155b151561031d57600080fd5b81600160008360ff1660ff168152602001908152602001600020819055508060ff167fe868bbbdd6cd2efcd9ba6e0129d43c349b0645524aba13f8a43bfc7c5ffb0889836040518082815260200191505060405180910390a25050565b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1681565b6000806000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16638b8414c46000604051602001526040518163ffffffff167c0100000000000000000000000000000000000000000000000000000000028152600401602060405180830381600087803b151561042f57600080fd5b6102c65a03f1151561044057600080fd5b5050506040518051905090508073ffffffffffffffffffffffffffffffffffffffff16630eaba26a6000604051602001526040518163ffffffff167c0100000000000000000000000000000000000000000000000000000000028152600401602060405180830381600087803b15156104b857600080fd5b6102c65a03f115156104c957600080fd5b5050506040518051905091505090565b60008060009054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff1663a3b3fff16000604051602001526040518163ffffffff167c0100000000000000000000000000000000000000000000000000000000028152600401602060405180830381600087803b151561056857600080fd5b6102c65a03f1151561057957600080fd5b50505060405180519050905090565b60006105b860016105aa600261059c61039f565b6105e590919063ffffffff16565b61060090919063ffffffff16565b905090565b60006105e06105ca61039f565b6105d261024a565b6105e590919063ffffffff16565b905090565b60008082848115156105f357fe5b0490508091505092915050565b600080828401905083811015151561061457fe5b80915050929150505600a165627a7a723058206b7eef2a57eb659d5e77e45ab5bc074e99c6a841921038cdb931e119c6aac46c0029",
from_address_hash: "0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca",
gas: 4_533_872,
gas_used: 382_953,
index: 0,
init:
"0x6060604052341561000f57600080fd5b60405160208061071a83398101604052808051906020019091905050806000806101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff1602179055506003600160006001600281111561007e57fe5b60ff1660ff168152602001908152602001600020819055506002600160006002808111156100a857fe5b60ff1660ff168152602001908152602001600020819055505061064a806100d06000396000f30060606040526004361061008e576000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff168063247b3210146100935780632ffdfc8a146100bc57806374294144146100f6578063ae4b1b5b14610125578063bf7370d11461017a578063d1104cb2146101a3578063eecd1079146101f8578063fcff021c14610221575b600080fd5b341561009e57600080fd5b6100a661024a565b6040518082815260200191505060405180910390f35b34156100c757600080fd5b6100e0600480803560ff16906020019091905050610253565b6040518082815260200191505060405180910390f35b341561010157600080fd5b610123600480803590602001909190803560ff16906020019091905050610276565b005b341561013057600080fd5b61013861037a565b604051808273ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16815260200191505060405180910390f35b341561018557600080fd5b61018d61039f565b6040518082815260200191505060405180910390f35b34156101ae57600080fd5b6101b66104d9565b604051808273ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16815260200191505060405180910390f35b341561020357600080fd5b61020b610588565b6040518082815260200191505060405180910390f35b341561022c57600080fd5b6102346105bd565b6040518082815260200191505060405180910390f35b600060c8905090565b6000600160008360ff1660ff168152602001908152602001600020549050919050565b61027e6104d9565b73ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff161415156102b757600080fd5b60008160ff161115156102c957600080fd5b6002808111156102d557fe5b60ff168160ff16111515156102e957600080fd5b6000821180156103125750600160008260ff1660ff168152602001908152602001600020548214155b151561031d57600080fd5b81600160008360ff1660ff168152602001908152602001600020819055508060ff167fe868bbbdd6cd2efcd9ba6e0129d43c349b0645524aba13f8a43bfc7c5ffb0889836040518082815260200191505060405180910390a25050565b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1681565b6000806000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16638b8414c46000604051602001526040518163ffffffff167c0100000000000000000000000000000000000000000000000000000000028152600401602060405180830381600087803b151561042f57600080fd5b6102c65a03f1151561044057600080fd5b5050506040518051905090508073ffffffffffffffffffffffffffffffffffffffff16630eaba26a6000604051602001526040518163ffffffff167c0100000000000000000000000000000000000000000000000000000000028152600401602060405180830381600087803b15156104b857600080fd5b6102c65a03f115156104c957600080fd5b5050506040518051905091505090565b60008060009054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff1663a3b3fff16000604051602001526040518163ffffffff167c0100000000000000000000000000000000000000000000000000000000028152600401602060405180830381600087803b151561056857600080fd5b6102c65a03f1151561057957600080fd5b50505060405180519050905090565b60006105b860016105aa600261059c61039f565b6105e590919063ffffffff16565b61060090919063ffffffff16565b905090565b60006105e06105ca61039f565b6105d261024a565b6105e590919063ffffffff16565b905090565b60008082848115156105f357fe5b0490508091505092915050565b600080828401905083811015151561061457fe5b80915050929150505600a165627a7a723058206b7eef2a57eb659d5e77e45ab5bc074e99c6a841921038cdb931e119c6aac46c0029000000000000000000000000862d67cb0773ee3f8ce7ea89b328ffea861ab3ef",
trace_address: [],
transaction_hash: "0x0fa6f723216dba694337f9bb37d8870725655bdf2573526a39454685659e39b1",
type: "create",
value: 0
created_contract_address_hash: created_contract_address_hash,
created_contract_code: created_contract_code,
from_address_hash: from_address_hash,
gas: gas,
gas_used: gas_used,
index: index,
init: init,
trace_address: trace_address,
transaction_hash: transaction_hash,
type: type,
value: value
}
]
}
end
@tag :no_geth
test "with all invalid transaction_params returns {:error, reasons}" do
assert EthereumJSONRPC.Parity.fetch_internal_transactions([
test "with all invalid transaction_params returns {:error, reasons}", %{
json_rpc_named_arguments: json_rpc_named_arguments
} do
if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do
expect(EthereumJSONRPC.Mox, :json_rpc, fn _json, _options ->
{:ok,
[
%{
id: 0,
error: %{
code: -32603,
message: "Internal error occurred: {}, this should not be the case with eth_call, most likely a bug."
}
}
]}
end)
end
assert EthereumJSONRPC.Parity.fetch_internal_transactions(
[
%{
block_number: 1,
hash_data: "0x0000000000000000000000000000000000000000000000000000000000000001"
}
]) ==
],
json_rpc_named_arguments
) ==
{:error,
[
%{
code: -32603,
data: %{
"blockNumber" => 1,
"code" => -32603,
"data" => "TransactionNotFound",
"message" =>
"Internal error occurred: {}, this should not be the case with eth_call, most likely a bug.",
"transactionHash" => "0x0000000000000000000000000000000000000000000000000000000000000001"
},
message:
"Internal error occurred: {}, this should not be the case with eth_call, most likely a bug."
}
]}
end
@tag :no_geth
test "with a mix of valid and invalid transaction_params returns {:error, reasons}" do
assert EthereumJSONRPC.Parity.fetch_internal_transactions([
test "with a mix of valid and invalid transaction_params returns {:error, reasons}", %{
json_rpc_named_arguments: json_rpc_named_arguments
} do
if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do
expect(EthereumJSONRPC.Mox, :json_rpc, fn _json, _options ->
{:ok,
[
%{
id: 0,
result: %{
"trace" => []
}
},
%{
id: 1,
result: %{
"trace" => []
}
},
%{
id: 2,
error: %{
code: -32603,
message: "Internal error occurred: {}, this should not be the case with eth_call, most likely a bug."
}
},
%{
id: 3,
error: %{
code: -32603,
message: "Internal error occurred: {}, this should not be the case with eth_call, most likely a bug."
}
}
]}
end)
end
assert EthereumJSONRPC.Parity.fetch_internal_transactions(
[
# start with :ok
%{
block_number: 1,
@ -71,42 +191,33 @@ defmodule EthereumJSONRPC.ParityTest do
block_number: 1,
hash_data: "0x0000000000000000000000000000000000000000000000000000000000000001"
},
# :error, :ok clause
%{
block_number: 35,
hash_data: "0x6b80a90c958fb5791a070929379ed6eb7a33ecdf9f9cafcada2f6803b3f25ec3"
},
# :error, :error clause
%{
block_number: 2,
hash_data: "0x0000000000000000000000000000000000000000000000000000000000000002"
}
]) ==
],
json_rpc_named_arguments
) ==
{:error,
[
%{
code: -32603,
data: %{
"blockNumber" => 1,
"code" => -32603,
"data" => "TransactionNotFound",
"message" =>
"Internal error occurred: {}, this should not be the case with eth_call, most likely a bug.",
"transactionHash" => "0x0000000000000000000000000000000000000000000000000000000000000001"
},
%{
"blockNumber" => 35,
"code" => -32603,
"data" => "TransactionNotFound",
"message" =>
"Internal error occurred: {}, this should not be the case with eth_call, most likely a bug.",
"transactionHash" => "0x6b80a90c958fb5791a070929379ed6eb7a33ecdf9f9cafcada2f6803b3f25ec3"
message:
"Internal error occurred: {}, this should not be the case with eth_call, most likely a bug."
},
%{
code: -32603,
data: %{
"blockNumber" => 2,
"code" => -32603,
"data" => "TransactionNotFound",
"message" =>
"Internal error occurred: {}, this should not be the case with eth_call, most likely a bug.",
"transactionHash" => "0x0000000000000000000000000000000000000000000000000000000000000002"
},
message:
"Internal error occurred: {}, this should not be the case with eth_call, most likely a bug."
}
]}
end

@ -1,80 +1,134 @@
defmodule EthereumJSONRPC.ReceiptsTest do
use ExUnit.Case, async: true
use EthereumJSONRPC.Case
import EthereumJSONRPC, only: [integer_to_quantity: 1]
import Mox
alias EthereumJSONRPC.Receipts
setup do
%{variant: EthereumJSONRPC.config(:variant)}
end
setup :verify_on_exit!
doctest Receipts
# These are integration tests that depend on the sokol chain being used. sokol can be used with the following config
#
# config :explorer, EthereumJSONRPC,
# trace_url: "https://sokol-trace.poa.network",
# url: "https://sokol.poa.network"
#
describe "fetch/1" do
test "with receipts and logs", %{variant: variant} do
case variant do
describe "fetch/2" do
test "with receipts and logs", %{json_rpc_named_arguments: json_rpc_named_arguments} do
%{
cumulative_gas_used: cumulative_gas_used,
gas_used: gas_used,
address_hash: address_hash,
block_number: block_number,
data: data,
index: index,
first_topic: first_topic,
status: status,
type: type,
transaction_hash: transaction_hash,
transaction_index: transaction_index
} =
case Keyword.fetch!(json_rpc_named_arguments, :variant) do
EthereumJSONRPC.Geth ->
assert {:ok,
%{
logs: [],
receipts: [
cumulative_gas_used: 884_322,
address_hash: "0x1e2fbe6be9eb39fc894d38be976111f332172d83",
block_number: 3_560_000,
data:
"0x00000000000000000000000033066f6a8adf2d4f5db193524b6fbae062ec0d110000000000000000000000000000000000000000000000000000000000001030",
index: 12,
first_topic: "0xf6db2bace4ac8277384553ad9603d045220a91fb2448ab6130d7a6f044f9a8cf",
gas_used: 106_025,
status: :error,
type: nil,
transaction_hash: "0xd3efddbbeb6ad8d8bb3f6b8c8fb6165567e9dd868013146bdbeb60953c82822a",
transaction_index: 17
}
EthereumJSONRPC.Parity ->
%{
cumulative_gas_used: 1_238_877,
gas_used: 21000,
cumulative_gas_used: 50450,
gas_used: 50450,
address_hash: "0x8bf38d4764929064f2d4d3a56520a76ab3df415b",
block_number: 37,
data: "0x000000000000000000000000862d67cb0773ee3f8ce7ea89b328ffea861ab3ef",
index: 0,
first_topic: "0x600bcf04a13e752d1e3670a5a9f1c21177ca2a93c6f5391d4f1298d098097c22",
status: :ok,
transaction_hash: "0x360fb62cc817093e5624468735803ea39cad719e5c68ca322bae6ba4f520756f",
transaction_index: 57
type: "mined",
transaction_hash: "0x53bd884872de3e488692881baeec262e7b95234d3965248c39fe992fffd433e5",
transaction_index: 0
}
]
}} =
Receipts.fetch([
end
if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do
native_status =
case status do
:ok -> "0x1"
:error -> "0x0"
end
expect(EthereumJSONRPC.Mox, :json_rpc, fn _json, _options ->
{:ok,
[
%{
id: 0,
result: %{
"cumulativeGasUsed" => integer_to_quantity(cumulative_gas_used),
"gasUsed" => integer_to_quantity(gas_used),
"logs" => [
%{
gas: 90000,
hash: "0x360fb62cc817093e5624468735803ea39cad719e5c68ca322bae6ba4f520756f"
"address" => address_hash,
"blockNumber" => integer_to_quantity(block_number),
"data" => data,
"logIndex" => integer_to_quantity(index),
"topics" => [first_topic],
"transactionHash" => transaction_hash,
"type" => type
}
],
"status" => native_status,
"transactionHash" => transaction_hash,
"transactionIndex" => integer_to_quantity(transaction_index)
}
}
])
]}
end)
end
EthereumJSONRPC.Parity ->
assert {:ok,
%{
logs: [
%{
address_hash: "0x8bf38d4764929064f2d4d3a56520a76ab3df415b",
data: "0x000000000000000000000000862d67cb0773ee3f8ce7ea89b328ffea861ab3ef",
first_topic: "0x600bcf04a13e752d1e3670a5a9f1c21177ca2a93c6f5391d4f1298d098097c22",
address_hash: ^address_hash,
block_number: ^block_number,
data: ^data,
first_topic: ^first_topic,
fourth_topic: nil,
index: 0,
index: ^index,
second_topic: nil,
third_topic: nil,
transaction_hash: "0x53bd884872de3e488692881baeec262e7b95234d3965248c39fe992fffd433e5",
type: "mined"
transaction_hash: ^transaction_hash
}
| _
],
receipts: [
%{
cumulative_gas_used: 50450,
gas_used: 50450,
status: :ok,
transaction_hash: "0x53bd884872de3e488692881baeec262e7b95234d3965248c39fe992fffd433e5",
transaction_index: 0
cumulative_gas_used: ^cumulative_gas_used,
gas_used: ^gas_used,
status: ^status,
transaction_hash: ^transaction_hash,
transaction_index: ^transaction_index
}
]
}} =
Receipts.fetch([
Receipts.fetch(
[
%{
gas: 50451,
hash: "0x53bd884872de3e488692881baeec262e7b95234d3965248c39fe992fffd433e5"
gas: 9000,
hash: transaction_hash
}
])
_ ->
raise ArgumentError, "Unsupported variant (#{variant})"
end
],
json_rpc_named_arguments
)
end
end
end

@ -1,10 +1,79 @@
defmodule EthereumJSONRPC.Case do
@moduledoc """
Adds `json_rpc_named_arguments` to context.
Reads `ETHEREUM_JSONRPC_TRANSPORT` environment variable to determine which module to use `:json_rpc_named_arguments`
`:transport`:
* `EthereumJSONRPC.HTTP` - Allow testing of HTTP-only behavior like status codes
* `EthereumJSONRPC.Mox` - mock, transport neutral responses. The default for local testing.
When `ETHEREUM_JSONRPC_TRANSPORT` is `EthereumJSONRPC.HTTP`, then reads `ETHEREUM_JSONRPC_HTTP_URL` environment
variable to determine `:json_rpc_named_arguments` `:transport_options` `:url`. Failure to set
`ETHEREUM_JSONRPC_HTTP_URL` in this case will raise an `ArgumentError`.
* `EthereumJSONRPC.HTTP.HTTPoison` - HTTP responses from calls to real chain URLs
* `EthereumJSONRPC.HTTP.Mox` - mock HTTP responses, so can be used for HTTP-only behavior like status codes.
"""
use ExUnit.CaseTemplate
require Logger
setup do
transport = transport()
transport_options =
case transport do
EthereumJSONRPC.HTTP ->
[
http: EthereumJSONRPC.HTTP.Case.http(),
url: EthereumJSONRPC.HTTP.Case.url(),
http_options: EthereumJSONRPC.HTTP.Case.http_options()
]
_ ->
[]
end
%{
json_rpc_named_arguments: [
transport: transport,
transport_options: transport_options,
variant: variant()
]
}
end
def log_bad_gateway(under_test, assertions) do
case under_test.() do
{:error, {:bad_gateway, url}} -> Logger.error(fn -> ["Bad Gateway to ", url, ". Check CloudFlare."] end)
other -> assertions.(other)
end
end
def module(environment_variable, default) do
alias =
environment_variable
|> System.get_env()
|> Kernel.||(default)
module = Module.concat([alias])
with {:error, reason} <- Code.ensure_loaded(module) do
raise ArgumentError,
"Could not load `#{environment_variable}` environment variable module (#{module}) due to #{reason}"
end
module
end
def transport do
module("ETHEREUM_JSONRPC_TRANSPORT", "EthereumJSONRPC.Mox")
end
def variant do
module("ETHEREUM_JSONRPC_VARIANT", "EthereumJSONRPC.Parity")
end
end

@ -0,0 +1,32 @@
defmodule EthereumJSONRPC.HTTP.Case do
use ExUnit.CaseTemplate
import EthereumJSONRPC.Case, only: [module: 2]
setup do
%{
json_rpc_named_arguments: [
transport: EthereumJSONRPC.HTTP,
transport_options: [
http: http(),
url: url(),
http_options: http_options()
]
]
}
end
def http do
module("ETHEREUM_JSONRPC_HTTP", "EthereumJSONRPC.HTTP.Mox")
end
def http_options do
[recv_timeout: 60_000, timeout: 60_000, hackney: [pool: :ethereum_jsonrpc]]
end
def url do
"ETHEREUM_JSONRPC_HTTP_URL"
|> System.get_env()
|> Kernel.||("https://example.com")
end
end

@ -6,5 +6,9 @@ File.mkdir_p!(junit_folder)
# Counter `test --no-start`. `--no-start` is needed for `:indexer` compatibility
{:ok, _} = Application.ensure_all_started(:ethereum_jsonrpc)
Mox.defmock(EthereumJSONRPC.Mox, for: EthereumJSONRPC.Transport)
# for when we need to simulate HTTP-specific stuff like 413 Request Entity Too Large
Mox.defmock(EthereumJSONRPC.HTTP.Mox, for: EthereumJSONRPC.HTTP)
ExUnit.configure(formatters: [JUnitFormatter, ExUnit.CLIFormatter])
ExUnit.start()

@ -2190,8 +2190,7 @@ defmodule Explorer.Chain do
|> Repo.insert()
end
@spec changes_list(params :: map, [{:for, module} | {:with, :atom}]) ::
{:ok, changes :: map} | {:error, [Changeset.t()]}
@spec changes_list(params :: [map], [{:for, module} | {:with, atom}]) :: {:ok, [map]} | {:error, [Changeset.t()]}
defp changes_list(params, options) when is_list(options) do
ecto_schema_module = Keyword.fetch!(options, :for)
changeset_function_name = Keyword.get(options, :with, :changeset)
@ -2395,7 +2394,7 @@ defmodule Explorer.Chain do
Repo.transaction(multi, timeout: Keyword.get(options, :timeout, @transaction_timeout))
end
@spec insert_internal_transactions([map()], [timestamps_option]) ::
@spec insert_internal_transactions([map], [timeout_option | timestamps_option]) ::
{:ok, [%{index: non_neg_integer, transaction_hash: Hash.t()}]}
| {:error, [Changeset.t()]}
defp insert_internal_transactions(changes_list, named_arguments)

@ -3,7 +3,9 @@
use Mix.Config
config :indexer,
block_rate: 5_000,
debug_logs: !!System.get_env("DEBUG_INDEXER")
debug_logs: !!System.get_env("DEBUG_INDEXER"),
ecto_repos: [Explorer.Repo]
config :indexer, ecto_repos: [Explorer.Repo]
# Import environment specific config. This must remain at the bottom
# of this file so it overrides the configuration defined above.
import_config "#{Mix.env()}.exs"

@ -0,0 +1,7 @@
use Mix.Config
variant = System.get_env("ETHEREUM_JSONRPC_VARIANT") || "parity"
# Import variant specific config. This must remain at the bottom
# of this file so it overrides the configuration defined above.
import_config "dev/#{variant}.exs"

@ -0,0 +1,13 @@
use Mix.Config
config :indexer,
block_rate: 5_000,
json_rpc_named_arguments: [
transport: EthereumJSONRPC.HTTP,
transport_options: [
http: EthereumJSONRPC.HTTP.HTTPoison,
url: "https://mainnet.infura.io/8lTvJTKmHPCHazkneJsY",
http_options: [recv_timeout: 60_000, timeout: 60_000, hackney: [pool: :ethereum_jsonrpc]]
],
variant: EthereumJSONRPC.Geth
]

@ -0,0 +1,17 @@
use Mix.Config
config :indexer,
block_rate: 5_000,
json_rpc_named_arguments: [
transport: EthereumJSONRPC.HTTP,
transport_options: [
http: EthereumJSONRPC.HTTP.HTTPoison,
url: "https://sokol.poa.network",
method_to_url: [
eth_getBalance: "https://sokol-trace.poa.network",
trace_replayTransaction: "https://sokol-trace.poa.network"
],
http_options: [recv_timeout: 60_000, timeout: 60_000, hackney: [pool: :ethereum_jsonrpc]]
],
variant: EthereumJSONRPC.Parity
]

@ -0,0 +1,7 @@
use Mix.Config
variant = System.get_env("ETHEREUM_JSONRPC_VARIANT") || "parity"
# Import variant specific config. This must remain at the bottom
# of this file so it overrides the configuration defined above.
import_config "dev/#{variant}.exs"

@ -0,0 +1,13 @@
use Mix.Config
config :indexer,
block_rate: 5_000,
json_rpc_named_arguments: [
transport: EthereumJSONRPC.HTTP,
transport_options: [
http: EthereumJSONRPC.HTTP.HTTPoison,
url: "https://mainnet.infura.io/8lTvJTKmHPCHazkneJsY",
http_options: [recv_timeout: 60_000, timeout: 60_000, hackney: [pool: :ethereum_jsonrpc]]
],
variant: EthereumJSONRPC.Geth
]

@ -0,0 +1,17 @@
use Mix.Config
config :indexer,
block_rate: 5_000,
json_rpc_named_arguments: [
transport: EthereumJSONRPC.HTTP,
transport_options: [
http: EthereumJSONRPC.HTTP.HTTPoison,
url: "https://sokol.poa.network",
method_to_url: [
eth_getBalance: "https://sokol-trace.poa.network",
trace_replayTransaction: "https://sokol-trace.poa.network"
],
http_options: [recv_timeout: 60_000, timeout: 60_000, hackney: [pool: :ethereum_jsonrpc]]
],
variant: EthereumJSONRPC.Parity
]

@ -32,12 +32,24 @@ defmodule Indexer.AddressBalanceFetcher do
@doc false
def child_spec(provided_opts) do
opts = Keyword.merge(@defaults, provided_opts)
{state, mergable_opts} = Keyword.pop(provided_opts, :json_rpc_named_arguments)
unless state do
raise ArgumentError,
":json_rpc_named_arguments must be provided to `#{__MODULE__}.child_spec " <>
"to allow for json_rpc calls when running."
end
opts =
@defaults
|> Keyword.merge(mergable_opts)
|> Keyword.put(:state, state)
Supervisor.child_spec({BufferedTask, {__MODULE__, opts}}, id: __MODULE__)
end
@impl BufferedTask
def init(initial, reducer) do
def init(initial, reducer, _) do
{:ok, final} =
Chain.stream_unfetched_addresses(initial, fn address_fields, acc ->
address_fields
@ -49,12 +61,12 @@ defmodule Indexer.AddressBalanceFetcher do
end
@impl BufferedTask
def run(params_list, _retries) do
def run(params_list, _retries, json_rpc_named_arguments) do
latest_params_list = latest_params_list(params_list)
Indexer.debug(fn -> "fetching #{length(latest_params_list)} balances" end)
case EthereumJSONRPC.fetch_balances(latest_params_list) do
case EthereumJSONRPC.fetch_balances(latest_params_list, json_rpc_named_arguments) do
{:ok, addresses_params} ->
{:ok, _} = Chain.update_balances(addresses_params)
:ok

@ -9,10 +9,12 @@ defmodule Indexer.Application do
@impl Application
def start(_type, _args) do
json_rpc_named_arguments = Application.fetch_env!(:indexer, :json_rpc_named_arguments)
children = [
{Task.Supervisor, name: Indexer.TaskSupervisor},
{AddressBalanceFetcher, name: AddressBalanceFetcher},
{PendingTransactionFetcher, name: PendingTransactionFetcher},
{AddressBalanceFetcher, name: AddressBalanceFetcher, json_rpc_named_arguments: json_rpc_named_arguments},
{PendingTransactionFetcher, name: PendingTransactionFetcher, json_rpc_named_arguments: json_rpc_named_arguments},
{InternalTransactionFetcher, name: InternalTransactionFetcher},
{BlockFetcher, []}
]

@ -36,8 +36,7 @@ defmodule Indexer.BlockFetcher do
## Options
Default options are pulled from application config under the
`:explorer, :indexer` keyspace. The follow options can be overridden:
Default options are pulled from application config under the :indexer` keyspace. The follow options can be overridden:
* `:blocks_batch_size` - The number of blocks to request in one call to the JSONRPC. Defaults to
`#{@blocks_batch_size}`. Block requests also include the transactions for those blocks. *These transactions
@ -70,6 +69,7 @@ defmodule Indexer.BlockFetcher do
:timer.send_interval(15_000, self(), :debug_count)
state = %{
json_rpc_named_arguments: Keyword.fetch!(opts, :json_rpc_named_arguments),
genesis_task: nil,
realtime_task: nil,
realtime_interval: opts[:block_rate] || @block_rate,
@ -162,13 +162,13 @@ defmodule Indexer.BlockFetcher do
defp fetch_transaction_receipts(_state, []), do: {:ok, %{logs: [], receipts: []}}
defp fetch_transaction_receipts(%{} = state, transaction_params) do
defp fetch_transaction_receipts(%{json_rpc_named_arguments: json_rpc_named_arguments} = state, transaction_params) do
debug(fn -> "fetching #{length(transaction_params)} transaction receipts" end)
stream_opts = [max_concurrency: state.receipts_concurrency, timeout: :infinity]
transaction_params
|> Enum.chunk_every(state.receipts_batch_size)
|> Task.async_stream(&EthereumJSONRPC.fetch_transaction_receipts(&1), stream_opts)
|> Task.async_stream(&EthereumJSONRPC.fetch_transaction_receipts(&1, json_rpc_named_arguments), stream_opts)
|> Enum.reduce_while({:ok, %{logs: [], receipts: []}}, fn
{:ok, {:ok, %{logs: logs, receipts: receipts}}}, {:ok, %{logs: acc_logs, receipts: acc_receipts}} ->
{:cont, {:ok, %{logs: acc_logs ++ logs, receipts: acc_receipts ++ receipts}}}
@ -181,8 +181,8 @@ defmodule Indexer.BlockFetcher do
end)
end
defp genesis_task(%{} = state) do
{:ok, latest_block_number} = EthereumJSONRPC.fetch_block_number_by_tag("latest")
defp genesis_task(%{json_rpc_named_arguments: json_rpc_named_arguments} = state) do
{:ok, latest_block_number} = EthereumJSONRPC.fetch_block_number_by_tag("latest", json_rpc_named_arguments)
missing_ranges = missing_block_number_ranges(state, latest_block_number..0)
count = Enum.count(missing_ranges)
@ -294,8 +294,8 @@ defmodule Indexer.BlockFetcher do
end)
end
defp realtime_task(%{} = state) do
{:ok, latest_block_number} = EthereumJSONRPC.fetch_block_number_by_tag("latest")
defp realtime_task(%{json_rpc_named_arguments: json_rpc_named_arguments} = state) do
{:ok, latest_block_number} = EthereumJSONRPC.fetch_block_number_by_tag("latest", json_rpc_named_arguments)
{:ok, seq} = Sequence.start_link(first: latest_block_number, step: 2)
stream_import(state, seq, max_concurrency: 1)
end
@ -313,8 +313,9 @@ defmodule Indexer.BlockFetcher do
# Run at state.blocks_concurrency max_concurrency when called by `stream_import/3`
# Only public for testing
@doc false
def import_range(range, %{} = state, seq) do
with {:blocks, {:ok, next, result}} <- {:blocks, EthereumJSONRPC.fetch_blocks_by_range(range)},
def import_range(range, %{json_rpc_named_arguments: json_rpc_named_arguments} = state, seq) do
with {:blocks, {:ok, next, result}} <-
{:blocks, EthereumJSONRPC.fetch_blocks_by_range(range, json_rpc_named_arguments)},
%{blocks: blocks, transactions: transactions_without_receipts} = result,
cap_seq(seq, next, range),
{:receipts, {:ok, receipt_params}} <-

@ -50,6 +50,31 @@ defmodule Indexer.BufferedTask do
require Logger
alias Indexer.BufferedTask
@enforce_keys [
:pid,
:callback_module,
:callback_module_state,
:task_supervisor,
:flush_interval,
:max_batch_size,
:init_chunk_size
]
defstruct pid: nil,
init_task: nil,
flush_timer: nil,
callback_module: nil,
callback_module_state: nil,
task_supervisor: nil,
flush_interval: nil,
max_batch_size: nil,
max_concurrency: nil,
init_chunk_size: nil,
current_buffer: [],
buffer: :queue.new(),
tasks: %{}
@typedoc """
Entry passed to `t:reducer/2` in `c:init/2` and grouped together into a list as `t:entries/0` passed to `c:run/2`.
"""
@ -78,21 +103,28 @@ defmodule Indexer.BufferedTask do
"""
@type reducer :: (entry, accumulator -> accumulator)
@typedoc """
Callback module controlled state. Can be used to store extra information needed for each `run/2`
"""
@type state :: term()
@doc """
Populates a task's buffer on boot with an initial set of entries.
For example, the following callback would buffer all unfetched account balances on startup:
def init(initial, reducer) do
Chain.stream_unfetched_addresses([:hash], initial, fn %{hash: hash}, acc ->
def init(initial, reducer, state) do
final = Chain.stream_unfetched_addresses([:hash], initial, fn %{hash: hash}, acc ->
reducer.(Hash.to_string(hash), acc)
end)
{final, state}
end
The `init/2` operation may be long-running as it is run in a separate process and allows concurrent calls to
`Explorer.BufferedTask.buffer/2` for on-demand entries.
"""
@callback init(initial, reducer) :: accumulator
@callback init(initial, reducer, state) :: accumulator
@doc """
Invoked as concurrency becomes available with a list of batched entries to be processed.
@ -116,7 +148,7 @@ defmodule Indexer.BufferedTask do
* `{:retry, new_entries :: list}` - run should be retried with `new_entries`
"""
@callback run(entries, retries :: pos_integer) :: :ok | :retry
@callback run(entries, retries :: pos_integer, state) :: :ok | :retry | {:retry, new_entries :: list}
@doc """
Buffers list of entries for future async execution.
@ -163,6 +195,7 @@ defmodule Indexer.BufferedTask do
| {:max_concurrency, pos_integer()}
| {:name, GenServer.name()}
| {:task_supervisor, GenServer.name()}
| {:state, state}
]}
) :: {:ok, pid()} | {:error, {:already_started, pid()}}
def start_link({module, base_opts}) do
@ -175,19 +208,15 @@ defmodule Indexer.BufferedTask do
def init({callback_module, opts}) do
send(self(), :initial_stream)
state = %{
state = %BufferedTask{
pid: self(),
init_task: nil,
flush_timer: nil,
callback_module: callback_module,
callback_module_state: Keyword.fetch!(opts, :state),
task_supervisor: Keyword.fetch!(opts, :task_supervisor),
flush_interval: Keyword.fetch!(opts, :flush_interval),
max_batch_size: Keyword.fetch!(opts, :max_batch_size),
max_concurrency: Keyword.fetch!(opts, :max_concurrency),
init_chunk_size: Keyword.fetch!(opts, :init_chunk_size),
current_buffer: [],
buffer: :queue.new(),
tasks: %{}
init_chunk_size: Keyword.fetch!(opts, :init_chunk_size)
}
{:ok, state}
@ -217,7 +246,7 @@ defmodule Indexer.BufferedTask do
{:noreply, state}
end
def handle_info({:DOWN, ref, :process, _pid, :normal}, %{init_task: ref} = state) do
def handle_info({:DOWN, ref, :process, _pid, :normal}, %BufferedTask{init_task: ref} = state) do
{:noreply, %{state | init_task: :complete}}
end
@ -245,7 +274,7 @@ defmodule Indexer.BufferedTask do
end
defp drop_task(state, ref) do
spawn_next_batch(%{state | tasks: Map.delete(state.tasks, ref)})
spawn_next_batch(%BufferedTask{state | tasks: Map.delete(state.tasks, ref)})
end
defp drop_task_and_retry(state, ref, new_batch \\ nil) do
@ -262,7 +291,7 @@ defmodule Indexer.BufferedTask do
%{state | current_buffer: [entries | state.current_buffer]}
end
defp queue_in_state(%{} = state, batch, retries) do
defp queue_in_state(%BufferedTask{} = state, batch, retries) do
%{state | buffer: queue_in_queue(state.buffer, batch, retries)}
end
@ -270,11 +299,14 @@ defmodule Indexer.BufferedTask do
:queue.in({batch, retries}, queue)
end
defp do_initial_stream(%{init_chunk_size: init_chunk_size} = state) do
defp do_initial_stream(
%BufferedTask{callback_module_state: callback_module_state, init_chunk_size: init_chunk_size} = state
) do
task =
Task.Supervisor.async(state.task_supervisor, fn ->
{0, []}
|> state.callback_module.init(fn
|> state.callback_module.init(
fn
entry, {len, acc} when len + 1 >= init_chunk_size ->
[entry | acc]
|> chunk_into_queue(state)
@ -284,11 +316,13 @@ defmodule Indexer.BufferedTask do
entry, {len, acc} ->
{len + 1, [entry | acc]}
end)
end,
callback_module_state
)
|> catchup_remaining(state)
end)
schedule_next_buffer_flush(%{state | init_task: task.ref})
schedule_next_buffer_flush(%BufferedTask{state | init_task: task.ref})
end
defp catchup_remaining({0, []}, _state), do: :ok
@ -330,7 +364,7 @@ defmodule Indexer.BufferedTask do
task =
Task.Supervisor.async_nolink(state.task_supervisor, fn ->
{:performed, state.callback_module.run(batch, retries)}
{:performed, state.callback_module.run(batch, retries, state.callback_module_state)}
end)
%{state | tasks: Map.put(state.tasks, task.ref, {batch, retries}), buffer: new_queue}
@ -339,11 +373,11 @@ defmodule Indexer.BufferedTask do
end
end
defp flush(%{current_buffer: []} = state) do
defp flush(%BufferedTask{current_buffer: []} = state) do
state |> spawn_next_batch() |> schedule_next_buffer_flush()
end
defp flush(%{current_buffer: current} = state) do
defp flush(%BufferedTask{current_buffer: current} = state) do
current
|> List.flatten()
|> Enum.chunk_every(state.max_batch_size)

@ -48,12 +48,24 @@ defmodule Indexer.InternalTransactionFetcher do
@doc false
def child_spec(provided_opts) do
opts = Keyword.merge(@defaults, provided_opts)
{state, mergable_opts} = Keyword.pop(provided_opts, :json_rpc_named_arguments)
unless state do
raise ArgumentError,
":json_rpc_named_arguments must be provided to `#{__MODULE__}.child_spec " <>
"to allow for json_rpc calls when running."
end
opts =
@defaults
|> Keyword.merge(mergable_opts)
|> Keyword.put(:state, state)
Supervisor.child_spec({BufferedTask, {__MODULE__, opts}}, id: __MODULE__)
end
@impl BufferedTask
def init(initial, reducer) do
def init(initial, reducer, _) do
{:ok, final} =
Chain.stream_transactions_with_unfetched_internal_transactions(
[:block_number, :hash],
@ -73,12 +85,12 @@ defmodule Indexer.InternalTransactionFetcher do
end
@impl BufferedTask
def run(transactions_params, _retries) do
def run(transactions_params, _retries, json_rpc_named_arguments) do
unique_transactions_params = unique_transactions_params(transactions_params)
Indexer.debug(fn -> "fetching internal transactions for #{length(unique_transactions_params)} transactions" end)
case EthereumJSONRPC.fetch_internal_transactions(unique_transactions_params) do
case EthereumJSONRPC.fetch_internal_transactions(unique_transactions_params, json_rpc_named_arguments) do
{:ok, internal_transactions_params} ->
addresses_params = AddressExtraction.extract_addresses(%{internal_transactions: internal_transactions_params})

@ -9,7 +9,7 @@ defmodule Indexer.PendingTransactionFetcher do
require Logger
import EthereumJSONRPC, only: [fetch_pending_transactions: 0]
import EthereumJSONRPC, only: [fetch_pending_transactions: 1]
alias Explorer.Chain
alias Indexer.{AddressExtraction, PendingTransactionFetcher}
@ -18,6 +18,7 @@ defmodule Indexer.PendingTransactionFetcher do
@default_interval 1_000
defstruct interval: @default_interval,
json_rpc_named_arguments: [],
task_ref: nil,
task_pid: nil
@ -35,6 +36,8 @@ defmodule Indexer.PendingTransactionFetcher do
* `:pending_transaction_interval` - the millisecond time between checking for pending transactions. Defaults to
`#{@default_interval}` milliseconds.
* `:spawn_opt` - if present, its value is passed as options to the underlying process as in `Process.spawn/4`
* `:json_rpc_named_arguments` - `t:EthereumJSONRPC.json_rpc_named_arguments/0` passed to
`EthereumJSONRPC.json_rpc/2`.
* `:timeout` - if present, the server is allowed to spend the given number of milliseconds initializing or it will
be terminated and the start function will return `{:error, :timeout}`
@ -51,7 +54,10 @@ defmodule Indexer.PendingTransactionFetcher do
|> Keyword.merge(opts)
state =
%PendingTransactionFetcher{interval: opts[:pending_transaction_interval] || @default_interval}
%PendingTransactionFetcher{
json_rpc_named_arguments: Keyword.fetch!(opts, :json_rpc_named_arguments),
interval: opts[:pending_transaction_interval] || @default_interval
}
|> schedule_fetch()
{:ok, state}
@ -84,8 +90,8 @@ defmodule Indexer.PendingTransactionFetcher do
state
end
defp task(%PendingTransactionFetcher{} = _state) do
case fetch_pending_transactions() do
defp task(%PendingTransactionFetcher{json_rpc_named_arguments: json_rpc_named_arguments} = _state) do
case fetch_pending_transactions(json_rpc_named_arguments) do
{:ok, transactions_params} ->
addresses_params = AddressExtraction.extract_addresses(%{transactions: transactions_params}, pending: true)

@ -39,7 +39,9 @@ defmodule Indexer.MixProject do
# JSONRPC access to Parity for `Explorer.Indexer`
{:ethereum_jsonrpc, in_umbrella: true},
# Importing to database
{:explorer, in_umbrella: true}
{:explorer, in_umbrella: true},
# Mocking `EthereumJSONRPC.Transport`, so we avoid hitting real chains for local testing
{:mox, "~> 0.3.2", only: [:test]}
]
end

@ -1,19 +1,35 @@
defmodule Indexer.AddressBalanceFetcherTest do
# MUST be `async: false` so that {:shared, pid} is set for connection to allow AddressBalanceFetcher's self-send to have
# connection allowed immediately.
use Explorer.DataCase, async: false
use EthereumJSONRPC.Case, async: false
use Explorer.DataCase
import EthereumJSONRPC, only: [integer_to_quantity: 1]
import Mox
alias Explorer.Chain.{Address, Hash, Wei}
alias Indexer.{AddressBalanceFetcher, AddressBalanceFetcherCase}
@moduletag :capture_log
# MUST use global mode because we aren't guaranteed to get `start_supervised`'s pid back fast enough to `allow` it to
# use expectations and stubs from test's pid.
setup :set_mox_global
setup :verify_on_exit!
setup do
start_supervised!({Task.Supervisor, name: Indexer.TaskSupervisor})
%{variant: EthereumJSONRPC.config(:variant)}
:ok
end
describe "init/1" do
test "fetches unfetched Block miner balance", %{variant: variant} do
test "fetches unfetched Block miner balance", %{
json_rpc_named_arguments: json_rpc_named_arguments
} do
variant = Keyword.fetch!(json_rpc_named_arguments, :variant)
%{block_number: block_number, fetched_balance: fetched_balance, miner_hash_data: miner_hash_data} =
case variant do
EthereumJSONRPC.Geth ->
@ -30,10 +46,20 @@ defmodule Indexer.AddressBalanceFetcherTest do
miner_hash_data: "0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca"
}
_ ->
variant ->
raise ArgumentError, "Unsupported variant (#{variant})"
end
if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do
block_quantity = integer_to_quantity(block_number)
EthereumJSONRPC.Mox
|> expect(:json_rpc, fn [%{id: id, method: "eth_getBalance", params: [^miner_hash_data, ^block_quantity]}],
_options ->
{:ok, [%{id: id, result: integer_to_quantity(fetched_balance)}]}
end)
end
{:ok, miner_hash} = Hash.Address.cast(miner_hash_data)
miner = insert(:address, hash: miner_hash)
block = insert(:block, miner: miner, number: block_number)
@ -41,7 +67,7 @@ defmodule Indexer.AddressBalanceFetcherTest do
assert miner.fetched_balance == nil
assert miner.fetched_balance_block_number == nil
AddressBalanceFetcherCase.start_supervised!()
AddressBalanceFetcherCase.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments)
fetched_address =
wait(fn ->
@ -54,7 +80,11 @@ defmodule Indexer.AddressBalanceFetcherTest do
assert fetched_address.fetched_balance_block_number == block.number
end
test "fetches unfetched addresses when less than max batch size", %{variant: variant} do
test "fetches unfetched addresses when less than max batch size", %{
json_rpc_named_arguments: json_rpc_named_arguments
} do
variant = Keyword.fetch!(json_rpc_named_arguments, :variant)
%{block_number: block_number, fetched_balance: fetched_balance, miner_hash_data: miner_hash_data} =
case variant do
EthereumJSONRPC.Geth ->
@ -71,15 +101,25 @@ defmodule Indexer.AddressBalanceFetcherTest do
miner_hash_data: "0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca"
}
_ ->
variant ->
raise ArgumentError, "Unsupported variant (#{variant})"
end
if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do
block_quantity = integer_to_quantity(block_number)
EthereumJSONRPC.Mox
|> expect(:json_rpc, fn [%{id: id, method: "eth_getBalance", params: [^miner_hash_data, ^block_quantity]}],
_options ->
{:ok, [%{id: id, result: integer_to_quantity(fetched_balance)}]}
end)
end
{:ok, miner_hash} = Hash.Address.cast(miner_hash_data)
miner = insert(:address, hash: miner_hash)
block = insert(:block, miner: miner, number: block_number)
AddressBalanceFetcherCase.start_supervised!(max_batch_size: 2)
AddressBalanceFetcherCase.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments, max_batch_size: 2)
fetched_address =
wait(fn ->
@ -94,8 +134,8 @@ defmodule Indexer.AddressBalanceFetcherTest do
end
describe "async_fetch_balances/1" do
test "fetches balances for address_hashes", %{variant: variant} do
AddressBalanceFetcherCase.start_supervised!()
test "fetches balances for address_hashes", %{json_rpc_named_arguments: json_rpc_named_arguments} do
variant = Keyword.fetch!(json_rpc_named_arguments, :variant)
%{block_number: block_number, fetched_balance: fetched_balance, hash: hash} =
case variant do
@ -120,10 +160,23 @@ defmodule Indexer.AddressBalanceFetcherTest do
}
}
_ ->
variant ->
raise ArgumentError, "Unsupported variant (#{variant})"
end
if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do
block_quantity = integer_to_quantity(block_number)
hash_data = to_string(hash)
EthereumJSONRPC.Mox
|> expect(:json_rpc, fn [%{id: id, method: "eth_getBalance", params: [^hash_data, ^block_quantity]}],
_options ->
{:ok, [%{id: id, result: integer_to_quantity(fetched_balance)}]}
end)
end
AddressBalanceFetcherCase.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments)
assert :ok = AddressBalanceFetcher.async_fetch_balances([%{block_number: block_number, hash: hash}])
address =
@ -137,10 +190,11 @@ defmodule Indexer.AddressBalanceFetcherTest do
end
describe "run/2" do
@tag capture_log: true
test "duplicate address hashes the max block_quantity", %{variant: variant} do
test "duplicate address hashes the max block_quantity", %{
json_rpc_named_arguments: json_rpc_named_arguments
} do
%{fetched_balance: fetched_balance, hash_data: hash_data} =
case variant do
case Keyword.fetch!(json_rpc_named_arguments, :variant) do
EthereumJSONRPC.Geth ->
%{
fetched_balance: 5_000_000_000_000_000_000,
@ -153,13 +207,21 @@ defmodule Indexer.AddressBalanceFetcherTest do
hash_data: "0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca"
}
_ ->
variant ->
raise ArgumentError, "Unsupported variant (#{variant})"
end
if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do
EthereumJSONRPC.Mox
|> expect(:json_rpc, fn [%{id: id, method: "eth_getBalance", params: [^hash_data, "0x2"]}], _options ->
{:ok, [%{id: id, result: integer_to_quantity(fetched_balance)}]}
end)
end
case AddressBalanceFetcher.run(
[%{block_quantity: "0x1", hash_data: hash_data}, %{block_quantity: "0x2", hash_data: hash_data}],
0
0,
json_rpc_named_arguments
) do
:ok ->
fetched_address = Repo.one!(from(address in Address, where: address.hash == ^hash_data))
@ -177,12 +239,20 @@ defmodule Indexer.AddressBalanceFetcherTest do
end
end
test "duplicate address hashes only retry max block_quantity" do
test "duplicate address hashes only retry max block_quantity", %{json_rpc_named_arguments: json_rpc_named_arguments} do
hash_data = "0x000000000000000000000000000000000"
if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do
EthereumJSONRPC.Mox
|> expect(:json_rpc, fn [%{id: id, method: "eth_getBalance", params: [^hash_data, "0x2"]}], _options ->
{:ok, [%{id: id, error: %{code: 404, message: "Not Found"}}]}
end)
end
assert AddressBalanceFetcher.run(
[%{block_quantity: "0x1", hash_data: hash_data}, %{block_quantity: "0x2", hash_data: hash_data}],
0
0,
json_rpc_named_arguments
) ==
{:retry,
[

@ -1,8 +1,11 @@
defmodule Indexer.BlockFetcherTest do
# `async: false` due to use of named GenServer
use Explorer.DataCase, async: false
use EthereumJSONRPC.Case, async: false
use Explorer.DataCase
import ExUnit.CaptureLog
import Mox
import EthereumJSONRPC, only: [integer_to_quantity: 1]
import EthereumJSONRPC.Case
alias Explorer.Chain.{Address, Block, Log, Transaction, Wei}
@ -17,7 +20,13 @@ defmodule Indexer.BlockFetcherTest do
Sequence
}
@tag capture_log: true
@moduletag capture_log: true
# MUST use global mode because we aren't guaranteed to get `start_supervised`'s pid back fast enough to `allow` it to
# use expectations and stubs from test's pid.
setup :set_mox_global
setup :verify_on_exit!
# First block with all schemas to import
# 37 is determined using the following query:
@ -37,13 +46,173 @@ defmodule Indexer.BlockFetcherTest do
# ON blocks.hash = transactions.block_hash) as blocks
@first_full_block_number 37
setup do
%{variant: EthereumJSONRPC.config(:variant)}
describe "start_link/1" do
test "starts fetching blocks from latest and goes down", %{json_rpc_named_arguments: json_rpc_named_arguments} do
if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do
case Keyword.fetch!(json_rpc_named_arguments, :variant) do
EthereumJSONRPC.Parity ->
block_number = 3_416_888
block_quantity = integer_to_quantity(block_number)
EthereumJSONRPC.Mox
|> stub(:json_rpc, fn
# latest block number to seed starting block number for genesis and realtime tasks
%{method: "eth_getBlockByNumber", params: ["latest", false]}, _options ->
{:ok,
%{
"author" => "0xe2ac1c6843a33f81ae4935e5ef1277a392990381",
"difficulty" => "0xfffffffffffffffffffffffffffffffe",
"extraData" => "0xd583010a068650617269747986312e32362e32826c69",
"gasLimit" => "0x7a1200",
"gasUsed" => "0x0",
"hash" => "0x627baabf5a17c0cfc547b6903ac5e19eaa91f30d9141be1034e3768f6adbc94e",
"logsBloom" =>
"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
"miner" => "0xe2ac1c6843a33f81ae4935e5ef1277a392990381",
"number" => block_quantity,
"parentHash" => "0x006edcaa1e6fde822908783bc4ef1ad3675532d542fce53537557391cfe34c3c",
"receiptsRoot" => "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
"sealFields" => [
"0x841240b30d",
"0xb84158bc4fa5891934bc94c5dca0301867ce4f35925ef46ea187496162668210bba61b4cda09d7e0dca2f1dd041fad498ced6697aeef72656927f52c55b630f2591c01"
],
"sha3Uncles" => "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347",
"signature" =>
"58bc4fa5891934bc94c5dca0301867ce4f35925ef46ea187496162668210bba61b4cda09d7e0dca2f1dd041fad498ced6697aeef72656927f52c55b630f2591c01",
"size" => "0x243",
"stateRoot" => "0x9a8111062667f7b162851a1cbbe8aece5ff12e761b3dcee93b787fcc12548cf7",
"step" => "306230029",
"timestamp" => "0x5b437f41",
"totalDifficulty" => "0x342337ffffffffffffffffffffffffed8d29bb",
"transactions" => [],
"transactionsRoot" => "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
"uncles" => []
}}
[%{method: "eth_getBlockByNumber", params: [_, true]} | _] = requests, _options ->
{:ok,
Enum.map(requests, fn %{id: id, params: [block_quantity, true]} ->
%{
id: id,
jsonrpc: "2.0",
result: %{
"author" => "0xe2ac1c6843a33f81ae4935e5ef1277a392990381",
"difficulty" => "0xfffffffffffffffffffffffffffffffe",
"extraData" => "0xd583010a068650617269747986312e32362e32826c69",
"gasLimit" => "0x7a1200",
"gasUsed" => "0x0",
"hash" =>
Explorer.Factory.block_hash()
|> to_string(),
"logsBloom" =>
"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
"miner" => "0xe2ac1c6843a33f81ae4935e5ef1277a392990381",
"number" => block_quantity,
"parentHash" =>
Explorer.Factory.block_hash()
|> to_string(),
"receiptsRoot" => "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
"sealFields" => [
"0x841240b30d",
"0xb84158bc4fa5891934bc94c5dca0301867ce4f35925ef46ea187496162668210bba61b4cda09d7e0dca2f1dd041fad498ced6697aeef72656927f52c55b630f2591c01"
],
"sha3Uncles" => "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347",
"signature" =>
"58bc4fa5891934bc94c5dca0301867ce4f35925ef46ea187496162668210bba61b4cda09d7e0dca2f1dd041fad498ced6697aeef72656927f52c55b630f2591c01",
"size" => "0x243",
"stateRoot" => "0x9a8111062667f7b162851a1cbbe8aece5ff12e761b3dcee93b787fcc12548cf7",
"step" => "306230029",
"timestamp" => "0x5b437f41",
"totalDifficulty" => "0x342337ffffffffffffffffffffffffed8d29bb",
"transactions" => [],
"transactionsRoot" => "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
"uncles" => []
}
}
end)}
[%{method: "eth_getBalance"} | _] = requests, _options ->
{:ok, Enum.map(requests, fn %{id: id} -> %{id: id, jsonrpc: "2.0", result: "0x0"} end)}
end)
EthereumJSONRPC.Geth ->
block_number = 5_950_901
block_quantity = integer_to_quantity(block_number)
EthereumJSONRPC.Mox
|> stub(:json_rpc, fn
%{method: "eth_getBlockByNumber", params: ["latest", false]}, _options ->
{:ok,
%{
"difficulty" => "0xc2550dc5bfc5d",
"extraData" => "0x65746865726d696e652d657538",
"gasLimit" => "0x7a121d",
"gasUsed" => "0x6cc04b",
"hash" => "0x71f484056fec687fd469989426c94c469ff08a28eae9a1865359d64557bb99f6",
"logsBloom" =>
"0x900840000041000850020000002800020800840900200210041006005028810880231200c1a0800001003a00011813005102000020800207080210000020014c00888640001040300c180008000084001000010018010040001118181400a06000280428024010081100015008080814141000644404040a8021101010040001001022000000000880420004008000180004000a01002080890010000a0601001a0000410244421002c0000100920100020004000020c10402004080008000203001000200c4001a000002000c0000000100200410090bc52e080900108230000110010082120200000004e01002000500001009e14001002051000040830080",
"miner" => "0xea674fdde714fd979de3edf0f56aa9716b898ec8",
"mixHash" => "0x555275cd0ab4c3b2fe3936843ee25bb67da05ef7dcf17216bc0e382d21d139a0",
"nonce" => "0xa49e42a024600113",
"number" => block_quantity,
"parentHash" => "0xb4357733c59cc6f785542d072a205f4e195f7198f544ea5e01c1b90ef0f914a5",
"receiptsRoot" => "0x17baf8de366fecc1be494bff245be6357ac60a5fe786099dba89983778c8421e",
"sha3Uncles" => "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347",
"size" => "0x6c7b",
"stateRoot" => "0x79345c692a0bf363e95c37750336c534309b3f3fe8b59712ac1527118070f488",
"timestamp" => "0x5b475377",
"totalDifficulty" => "0x120258e22c69502fc88",
"transactions" => ["0xa4b58d1d1473f4891d9ff91f624dba73611bf1f6e9a60d3ca2dcfc75d2ab185c"],
"transactionsRoot" => "0x5972b7988f667d7e86679322641117e503ea2c1bc5a27822a8a8120fe53f2c8b",
"uncles" => []
}}
[%{method: "eth_getBlockByNumber", params: [_, true]} | _] = requests, _options ->
{:ok,
Enum.map(requests, fn %{id: id, params: [block_quantity, true]} ->
%{
id: id,
jsonrpc: "2.0",
result: %{
"difficulty" => "0xc22479024e55f",
"extraData" => "0x73656f3130",
"gasLimit" => "0x7a121d",
"gasUsed" => "0x7a0527",
"hash" =>
Explorer.Factory.block_hash()
|> to_string(),
"logsBloom" =>
"0x006a044c050a6759208088200009808898246808402123144ac15801c09a2672990130000042500000cc6090b063f195352095a88018194112101a02640000a0109c03c40568440b853a800a60044408604bb49d1d604c802008000884520208496608a520992e0f4b41a94188088920c1995107db4696c03839a911500084001009884100605084c4542953b08101103080254c34c802a00042a62f811340400d22080d000c0e39927ca481800c8024048425462000150850500205a224810041904023a80c00dc01040203000086020111210403081096822008c12500a2060a54834800400851210122c481a04a24b5284e9900a08110c180011001c03100",
"miner" => "0xb2930b35844a230f00e51431acae96fe543a0347",
"mixHash" => "0x5e07a58028d2cee7ddbefe245e6d7b5232d997b66cc906b18ad9ad51535ced24",
"nonce" => "0x3d88ebe8031aadf6",
"number" => block_quantity,
"parentHash" =>
Explorer.Factory.block_hash()
|> to_string(),
"receiptsRoot" => "0x5294a8b56be40c0c198aa443664e801bb926d49878f96151849f3ddd0cb5e76d",
"sha3Uncles" => "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347",
"size" => "0x4796",
"stateRoot" => "0x3755d4b5c9ae3cd58d7a856a46fbe8fb69f0ba93d81e831cd68feb8b61bc3009",
"timestamp" => "0x5b475393",
"totalDifficulty" => "0x120259a450e2527e1e7",
"transactions" => [],
"transactionsRoot" => "0xa71969ed649cd1f21846ab7b4029e79662941cc34cd473aa4590e666920ad2f4",
"uncles" => []
}
}
end)}
[%{method: "eth_getBalance"} | _] = requests, _options ->
{:ok, Enum.map(requests, fn %{id: id} -> %{id: id, jsonrpc: "2.0", result: "0x0"} end)}
end)
variant_name ->
raise ArgumentError, "Unsupported variant name (#{variant_name})"
end
end
describe "start_link/1" do
test "starts fetching blocks from latest and goes down" do
{:ok, latest_block_number} = EthereumJSONRPC.fetch_block_number_by_tag("latest")
{:ok, latest_block_number} = EthereumJSONRPC.fetch_block_number_by_tag("latest", json_rpc_named_arguments)
default_blocks_batch_size = BlockFetcher.default_blocks_batch_size()
@ -52,9 +221,9 @@ defmodule Indexer.BlockFetcherTest do
assert Repo.aggregate(Block, :count, :hash) == 0
start_supervised!({Task.Supervisor, name: Indexer.TaskSupervisor})
AddressBalanceFetcherCase.start_supervised!()
InternalTransactionFetcherCase.start_supervised!()
start_supervised!(BlockFetcher)
AddressBalanceFetcherCase.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments)
InternalTransactionFetcherCase.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments)
start_supervised!({BlockFetcher, json_rpc_named_arguments: json_rpc_named_arguments})
wait_for_results(fn ->
Repo.one!(from(block in Block, where: block.number == ^latest_block_number))
@ -93,10 +262,10 @@ defmodule Indexer.BlockFetcherTest do
@tag :capture_log
@heading "persisted counts"
test "without debug_logs", %{state: state} do
test "without debug_logs", %{json_rpc_named_arguments: json_rpc_named_arguments, state: state} do
start_supervised!({Task.Supervisor, name: Indexer.TaskSupervisor})
AddressBalanceFetcherCase.start_supervised!()
InternalTransactionFetcherCase.start_supervised!()
AddressBalanceFetcherCase.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments)
InternalTransactionFetcherCase.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments)
wait_for_tasks(InternalTransactionFetcher)
wait_for_tasks(AddressBalanceFetcher)
@ -108,10 +277,10 @@ defmodule Indexer.BlockFetcherTest do
end
@tag :capture_log
test "with debug_logs", %{state: state} do
test "with debug_logs", %{json_rpc_named_arguments: json_rpc_named_arguments, state: state} do
start_supervised!({Task.Supervisor, name: Indexer.TaskSupervisor})
AddressBalanceFetcherCase.start_supervised!()
InternalTransactionFetcherCase.start_supervised!()
AddressBalanceFetcherCase.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments)
InternalTransactionFetcherCase.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments)
wait_for_tasks(InternalTransactionFetcher)
wait_for_tasks(AddressBalanceFetcher)
@ -133,20 +302,135 @@ defmodule Indexer.BlockFetcherTest do
describe "import_range/3" do
setup :state
setup do
setup %{json_rpc_named_arguments: json_rpc_named_arguments} do
start_supervised!({Task.Supervisor, name: Indexer.TaskSupervisor})
AddressBalanceFetcherCase.start_supervised!()
InternalTransactionFetcherCase.start_supervised!()
{:ok, state} = BlockFetcher.init([])
AddressBalanceFetcherCase.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments)
InternalTransactionFetcherCase.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments)
{:ok, state} = BlockFetcher.init(json_rpc_named_arguments: json_rpc_named_arguments)
%{state: state}
end
test "with single element range that is valid imports one block", %{state: state, variant: variant} do
test "with single element range that is valid imports one block", %{
json_rpc_named_arguments: json_rpc_named_arguments,
state: state
} do
block_number = 0
if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do
block_quantity = integer_to_quantity(block_number)
miner_hash = "0x0000000000000000000000000000000000000000"
case Keyword.fetch!(json_rpc_named_arguments, :variant) do
EthereumJSONRPC.Parity ->
EthereumJSONRPC.Mox
|> expect(:json_rpc, fn [%{id: id, method: "eth_getBlockByNumber", params: [^block_quantity, true]}],
_options ->
{:ok,
[
%{
id: id,
jsonrpc: "2.0",
result: %{
"author" => "0x0000000000000000000000000000000000000000",
"difficulty" => "0x20000",
"extraData" => "0x",
"gasLimit" => "0x663be0",
"gasUsed" => "0x0",
"hash" => "0x5b28c1bfd3a15230c9a46b399cd0f9a6920d432e85381cc6a140b06e8410112f",
"logsBloom" =>
"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
"miner" => miner_hash,
"number" => block_quantity,
"parentHash" => "0x0000000000000000000000000000000000000000000000000000000000000000",
"receiptsRoot" => "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
"sealFields" => [
"0x80",
"0xb8410000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"
],
"sha3Uncles" => "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347",
"signature" =>
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
"size" => "0x215",
"stateRoot" => "0xfad4af258fd11939fae0c6c6eec9d340b1caac0b0196fd9a1bc3f489c5bf00b3",
"step" => "0",
"timestamp" => "0x0",
"totalDifficulty" => "0x20000",
"transactions" => [],
"transactionsRoot" => "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
"uncles" => []
}
}
]}
end)
|> expect(:json_rpc, fn [
%{
id: id,
jsonrpc: "2.0",
method: "eth_getBalance",
params: [^miner_hash, ^block_quantity]
}
],
_options ->
{:ok, [%{id: id, jsonrpc: "2.0", result: "0x0"}]}
end)
EthereumJSONRPC.Geth ->
EthereumJSONRPC.Mox
|> expect(:json_rpc, fn [%{id: id, method: "eth_getBlockByNumber", params: [^block_quantity, true]}],
_options ->
{:ok,
[
%{
id: id,
jsonrpc: "2.0",
result: %{
"difficulty" => "0x400000000",
"extraData" => "0x11bbe8db4e347b4e8c937c1c8370e4b5ed33adb3db69cbdb7a38e1e50b1b82fa",
"gasLimit" => "0x1388",
"gasUsed" => "0x0",
"hash" => "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3",
"logsBloom" =>
"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
"miner" => miner_hash,
"mixHash" => "0x0000000000000000000000000000000000000000000000000000000000000000",
"nonce" => "0x0000000000000042",
"number" => block_quantity,
"parentHash" => "0x0000000000000000000000000000000000000000000000000000000000000000",
"receiptsRoot" => "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
"sha3Uncles" => "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347",
"size" => "0x21c",
"stateRoot" => "0xd7f8974fb5ac78d9ac099b9ad5018bedc2ce0a72dad1827a1709da30580f0544",
"timestamp" => "0x0",
"totalDifficulty" => "0x400000000",
"transactions" => [],
"transactionsRoot" => "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
"uncles" => []
}
}
]}
end)
|> expect(:json_rpc, fn [
%{
id: id,
jsonrpc: "2.0",
method: "eth_getBalance",
params: [^miner_hash, ^block_quantity]
}
],
_options ->
{:ok, [%{id: id, jsonrpc: "2.0", result: "0x0"}]}
end)
variant_name ->
raise ArgumentError, "Unsupported variant name (#{variant_name})"
end
end
{:ok, sequence} = Sequence.start_link(first: 0, step: 1)
%{address_hash: address_hash, block_hash: block_hash} =
case variant do
case Keyword.fetch!(json_rpc_named_arguments, :variant) do
EthereumJSONRPC.Geth ->
%{
address_hash: %Explorer.Chain.Hash{
@ -175,12 +459,12 @@ defmodule Indexer.BlockFetcherTest do
}
}
_ ->
raise ArgumenrError, "Unsupported variant (#{variant})"
variant ->
raise ArgumentError, "Unsupported variant (#{variant})"
end
log_bad_gateway(
fn -> BlockFetcher.import_range(0..0, state, sequence) end,
fn -> BlockFetcher.import_range(block_number..block_number, state, sequence) end,
fn result ->
assert {:ok,
%{
@ -204,10 +488,185 @@ defmodule Indexer.BlockFetcherTest do
)
end
test "can import range with all synchronous imported schemas", %{state: state, variant: variant} do
# We can't currently index the whole Ethereum Mainnet, so we don't know what is the first full block.
# Implement when a full block is found for Ethereum Mainnet and remove :no_geth tag
@tag :no_geth
test "can import range with all synchronous imported schemas", %{
json_rpc_named_arguments: json_rpc_named_arguments,
state: state
} do
block_number = @first_full_block_number
if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do
case Keyword.fetch!(json_rpc_named_arguments, :variant) do
EthereumJSONRPC.Parity ->
block_quantity = integer_to_quantity(block_number)
from_address_hash = "0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca"
to_address_hash = "0x8bf38d4764929064f2d4d3a56520a76ab3df415b"
transaction_hash = "0x53bd884872de3e488692881baeec262e7b95234d3965248c39fe992fffd433e5"
EthereumJSONRPC.Mox
|> expect(:json_rpc, fn json, _options ->
assert [%{id: id, method: "eth_getBlockByNumber", params: [^block_quantity, true]}] = json
{:ok,
[
%{
id: id,
jsonrpc: "2.0",
result: %{
"author" => "0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca",
"difficulty" => "0xfffffffffffffffffffffffffffffffe",
"extraData" => "0xd5830108048650617269747986312e32322e31826c69",
"gasLimit" => "0x69fe20",
"gasUsed" => "0xc512",
"hash" => "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd",
"logsBloom" =>
"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000400000000000000000000000000000000000000000000000200000000000000000000020000000000000000200000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
"miner" => "0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca",
"number" => "0x25",
"parentHash" => "0xc37bbad7057945d1bf128c1ff009fb1ad632110bf6a000aac025a80f7766b66e",
"receiptsRoot" => "0xd300311aab7dcc98c05ac3f1893629b2c9082c189a0a0c76f4f63e292ac419d5",
"sealFields" => [
"0x84120a71de",
"0xb841fcdb570511ec61edda93849bb7c6b3232af60feb2ea74e4035f0143ab66dfdd00f67eb3eda1adddbb6b572db1e0abd39ce00f9b3ccacb9f47973279ff306fe5401"
],
"sha3Uncles" => "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347",
"signature" =>
"fcdb570511ec61edda93849bb7c6b3232af60feb2ea74e4035f0143ab66dfdd00f67eb3eda1adddbb6b572db1e0abd39ce00f9b3ccacb9f47973279ff306fe5401",
"size" => "0x2cf",
"stateRoot" => "0x2cd84079b0d0c267ed387e3895fd1c1dc21ff82717beb1132adac64276886e19",
"step" => "302674398",
"timestamp" => "0x5a343956",
"totalDifficulty" => "0x24ffffffffffffffffffffffffedf78dfd",
"transactions" => [
%{
"blockHash" => "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd",
"blockNumber" => "0x25",
"chainId" => "0x4d",
"condition" => nil,
"creates" => nil,
"from" => from_address_hash,
"gas" => "0x47b760",
"gasPrice" => "0x174876e800",
"hash" => transaction_hash,
"input" => "0x10855269000000000000000000000000862d67cb0773ee3f8ce7ea89b328ffea861ab3ef",
"nonce" => "0x4",
"publicKey" =>
"0xe5d196ad4ceada719d9e592f7166d0c75700f6eab2e3c3de34ba751ea786527cb3f6eb96ad9fdfdb9989ff572df50f1c42ef800af9c5207a38b929aff969b5c9",
"r" => "0xa7f8f45cce375bb7af8750416e1b03e0473f93c256da2285d1134fc97a700e01",
"raw" =>
"0xf88a0485174876e8008347b760948bf38d4764929064f2d4d3a56520a76ab3df415b80a410855269000000000000000000000000862d67cb0773ee3f8ce7ea89b328ffea861ab3ef81bea0a7f8f45cce375bb7af8750416e1b03e0473f93c256da2285d1134fc97a700e01a01f87a076f13824f4be8963e3dffd7300dae64d5f23c9a062af0c6ead347c135f",
"s" => "0x1f87a076f13824f4be8963e3dffd7300dae64d5f23c9a062af0c6ead347c135f",
"standardV" => "0x1",
"to" => to_address_hash,
"transactionIndex" => "0x0",
"v" => "0xbe",
"value" => "0x0"
}
],
"transactionsRoot" => "0x68e314a05495f390f9cd0c36267159522e5450d2adf254a74567b452e767bf34",
"uncles" => []
}
}
]}
end)
|> expect(:json_rpc, fn json, _options ->
assert [
%{
id: id,
method: "eth_getTransactionReceipt",
params: ["0x53bd884872de3e488692881baeec262e7b95234d3965248c39fe992fffd433e5"]
}
] = json
{:ok,
[
%{
id: id,
jsonrpc: "2.0",
result: %{
"blockHash" => "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd",
"blockNumber" => "0x25",
"contractAddress" => nil,
"cumulativeGasUsed" => "0xc512",
"gasUsed" => "0xc512",
"logs" => [
%{
"address" => "0x8bf38d4764929064f2d4d3a56520a76ab3df415b",
"blockHash" => "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd",
"blockNumber" => "0x25",
"data" => "0x000000000000000000000000862d67cb0773ee3f8ce7ea89b328ffea861ab3ef",
"logIndex" => "0x0",
"topics" => ["0x600bcf04a13e752d1e3670a5a9f1c21177ca2a93c6f5391d4f1298d098097c22"],
"transactionHash" => "0x53bd884872de3e488692881baeec262e7b95234d3965248c39fe992fffd433e5",
"transactionIndex" => "0x0",
"transactionLogIndex" => "0x0",
"type" => "mined"
}
],
"logsBloom" =>
"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000400000000000000000000000000000000000000000000000200000000000000000000020000000000000000200000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
"root" => nil,
"status" => "0x1",
"transactionHash" => "0x53bd884872de3e488692881baeec262e7b95234d3965248c39fe992fffd433e5",
"transactionIndex" => "0x0"
}
}
]}
end)
# async requests need to be grouped in one expect because the order is non-deterministic while multiple expect
# calls on the same name/arity are used in order
|> expect(:json_rpc, 5, fn json, _options ->
[request] = json
case request do
%{id: id, method: "eth_getBalance", params: [^to_address_hash, ^block_quantity]} ->
{:ok, [%{id: id, jsonrpc: "2.0", result: "0x1"}]}
%{id: id, method: "eth_getBalance", params: [^from_address_hash, ^block_quantity]} ->
{:ok, [%{id: id, jsonrpc: "2.0", result: "0xd0d4a965ab52d8cd740000"}]}
%{id: id, method: "trace_replayTransaction", params: [^transaction_hash, ["trace"]]} ->
{:ok,
[
%{
id: id,
jsonrpc: "2.0",
result: %{
"output" => "0x",
"stateDiff" => nil,
"trace" => [
%{
"action" => %{
"callType" => "call",
"from" => from_address_hash,
"gas" => "0x475ec8",
"input" => "0x10855269000000000000000000000000862d67cb0773ee3f8ce7ea89b328ffea861ab3ef",
"to" => to_address_hash,
"value" => "0x0"
},
"result" => %{"gasUsed" => "0x6c7a", "output" => "0x"},
"subtraces" => 0,
"traceAddress" => [],
"type" => "call"
}
],
"vmTrace" => nil
}
}
]}
end
end)
variant ->
raise ArgumentError, "Unsupported variant (#{variant})"
end
end
{:ok, sequence} = Sequence.start_link(first: 0, step: 1)
case variant do
case Keyword.fetch!(json_rpc_named_arguments, :variant) do
EthereumJSONRPC.Geth ->
block_number = 48230
@ -344,7 +803,7 @@ defmodule Indexer.BlockFetcherTest do
57, 101, 36, 140, 57, 254, 153, 47, 255, 212, 51, 229>>
}
]
}} = BlockFetcher.import_range(@first_full_block_number..@first_full_block_number, state, sequence)
}} = BlockFetcher.import_range(block_number..block_number, state, sequence)
wait_for_tasks(InternalTransactionFetcher)
wait_for_tasks(AddressBalanceFetcher)
@ -357,14 +816,14 @@ defmodule Indexer.BlockFetcherTest do
first_address = Repo.get!(Address, first_address_hash)
assert first_address.fetched_balance == %Wei{value: Decimal.new(1)}
assert first_address.fetched_balance_block_number == @first_full_block_number
assert first_address.fetched_balance_block_number == block_number
second_address = Repo.get!(Address, second_address_hash)
assert second_address.fetched_balance == %Wei{value: Decimal.new(252_460_837_000_000_000_000_000_000)}
assert second_address.fetched_balance_block_number == @first_full_block_number
assert second_address.fetched_balance_block_number == block_number
_ ->
variant ->
raise ArgumentError, "Unsupport variant (#{variant})"
end
end
@ -395,8 +854,8 @@ defmodule Indexer.BlockFetcherTest do
return
end
defp state(_) do
{:ok, state} = BlockFetcher.init([])
defp state(%{json_rpc_named_arguments: json_rpc_named_arguments}) do
{:ok, state} = BlockFetcher.init(json_rpc_named_arguments: json_rpc_named_arguments)
%{state: state}
end

@ -11,6 +11,7 @@ defmodule Indexer.BufferedTaskTest do
start_supervised(
{BufferedTask,
{callback_module,
state: nil,
task_supervisor: BufferedTaskSup,
flush_interval: 50,
max_batch_size: @max_batch_size,
@ -24,11 +25,11 @@ defmodule Indexer.BufferedTaskTest do
def initial_collection, do: for(i <- 1..11, do: "#{i}")
def init(initial, reducer) do
def init(initial, reducer, _state) do
Enum.reduce(initial_collection(), initial, fn item, acc -> reducer.(item, acc) end)
end
def run(batch, 0) do
def run(batch, 0, _state) do
send(__MODULE__, {:run, batch})
:ok
end
@ -37,11 +38,11 @@ defmodule Indexer.BufferedTaskTest do
defmodule EmptyTask do
@behaviour BufferedTask
def init(initial, _reducer) do
def init(initial, _reducer, _state) do
initial
end
def run(batch, 0) do
def run(batch, 0, _state) do
send(__MODULE__, {:run, batch})
:ok
end
@ -50,31 +51,31 @@ defmodule Indexer.BufferedTaskTest do
defmodule RetryableTask do
@behaviour BufferedTask
def init(initial, _reducer) do
def init(initial, _reducer, _state) do
initial
end
def run([:boom], 0) do
def run([:boom], 0, _state) do
send(__MODULE__, {:run, {0, :boom}})
raise "boom"
end
def run([:boom], 1) do
def run([:boom], 1, _state) do
send(__MODULE__, {:run, {1, :boom}})
:ok
end
def run([{:sleep, time}], _) do
def run([{:sleep, time}], _, _state) do
:timer.sleep(time)
:ok
end
def run(batch, retries) when retries < 2 do
def run(batch, retries, _state) when retries < 2 do
send(__MODULE__, {:run, {retries, batch}})
:retry
end
def run(batch, retries) do
def run(batch, retries, _state) do
send(__MODULE__, {:final_run, {retries, batch}})
:ok
end

@ -1,39 +1,94 @@
defmodule Indexer.InternalTransactionFetcherTest do
use Explorer.DataCase, async: false
use EthereumJSONRPC.Case, async: false
use Explorer.DataCase
import ExUnit.CaptureLog
import Mox
alias Indexer.{AddressBalanceFetcherCase, InternalTransactionFetcher}
alias Indexer.{AddressBalanceFetcherCase, InternalTransactionFetcher, PendingTransactionFetcher}
@moduletag :capture_log
# MUST use global mode because we aren't guaranteed to get PendingTransactionFetcher's pid back fast enough to `allow`
# it to use expectations and stubs from test's pid.
setup :set_mox_global
setup :verify_on_exit!
@moduletag [capture_log: true, no_geth: true]
test "does not try to fetch pending transactions from Indexer.PendingTransactionFetcher", %{
json_rpc_named_arguments: json_rpc_named_arguments
} do
if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do
case Keyword.fetch!(json_rpc_named_arguments, :variant) do
EthereumJSONRPC.Parity ->
EthereumJSONRPC.Mox
|> expect(:json_rpc, fn _json, _options ->
{:ok,
[
%{
"blockHash" => nil,
"blockNumber" => nil,
"chainId" => "0x4d",
"condition" => nil,
"creates" => "0xffc87239eb0267bc3ca2cd51d12fbf278e02ccb4",
"from" => "0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca",
"gas" => "0x47b760",
"gasPrice" => "0x174876e800",
"hash" => "0x3a3eb134e6792ce9403ea4188e5e79693de9e4c94e499db132be086400da79e6",
"input" =>
"0x6060604052341561000f57600080fd5b336000806101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff1602179055506102db8061005e6000396000f300606060405260043610610062576000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff1680630900f01014610067578063445df0ac146100a05780638da5cb5b146100c9578063fdacd5761461011e575b600080fd5b341561007257600080fd5b61009e600480803573ffffffffffffffffffffffffffffffffffffffff16906020019091905050610141565b005b34156100ab57600080fd5b6100b3610224565b6040518082815260200191505060405180910390f35b34156100d457600080fd5b6100dc61022a565b604051808273ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16815260200191505060405180910390f35b341561012957600080fd5b61013f600480803590602001909190505061024f565b005b60008060009054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff161415610220578190508073ffffffffffffffffffffffffffffffffffffffff1663fdacd5766001546040518263ffffffff167c010000000000000000000000000000000000000000000000000000000002815260040180828152602001915050600060405180830381600087803b151561020b57600080fd5b6102c65a03f1151561021c57600080fd5b5050505b5050565b60015481565b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1681565b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff1614156102ac57806001819055505b505600a165627a7a72305820a9c628775efbfbc17477a472413c01ee9b33881f550c59d21bee9928835c854b0029",
"nonce" => "0x0",
"publicKey" =>
"0xe5d196ad4ceada719d9e592f7166d0c75700f6eab2e3c3de34ba751ea786527cb3f6eb96ad9fdfdb9989ff572df50f1c42ef800af9c5207a38b929aff969b5c9",
"r" => "0xad3733df250c87556335ffe46c23e34dbaffde93097ef92f52c88632a40f0c75",
"raw" =>
"0xf9038d8085174876e8008347b7608080b903396060604052341561000f57600080fd5b336000806101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff1602179055506102db8061005e6000396000f300606060405260043610610062576000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff1680630900f01014610067578063445df0ac146100a05780638da5cb5b146100c9578063fdacd5761461011e575b600080fd5b341561007257600080fd5b61009e600480803573ffffffffffffffffffffffffffffffffffffffff16906020019091905050610141565b005b34156100ab57600080fd5b6100b3610224565b6040518082815260200191505060405180910390f35b34156100d457600080fd5b6100dc61022a565b604051808273ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16815260200191505060405180910390f35b341561012957600080fd5b61013f600480803590602001909190505061024f565b005b60008060009054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff161415610220578190508073ffffffffffffffffffffffffffffffffffffffff1663fdacd5766001546040518263ffffffff167c010000000000000000000000000000000000000000000000000000000002815260040180828152602001915050600060405180830381600087803b151561020b57600080fd5b6102c65a03f1151561021c57600080fd5b5050505b5050565b60015481565b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1681565b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff1614156102ac57806001819055505b505600a165627a7a72305820a9c628775efbfbc17477a472413c01ee9b33881f550c59d21bee9928835c854b002981bda0ad3733df250c87556335ffe46c23e34dbaffde93097ef92f52c88632a40f0c75a072caddc0371451a58de2ca6ab64e0f586ccdb9465ff54e1c82564940e89291e3",
"s" => "0x72caddc0371451a58de2ca6ab64e0f586ccdb9465ff54e1c82564940e89291e3",
"standardV" => "0x0",
"to" => nil,
"transactionIndex" => nil,
"v" => "0xbd",
"value" => "0x0"
}
]}
end)
|> stub(:json_rpc, fn _json, _options ->
{:ok, []}
end)
variant_name ->
raise ArgumentError, "Unsupported variant name (#{variant_name})"
end
end
@tag :no_geth
test "does not try to fetch pending transactions from Indexer.PendingTransactionFetcher" do
start_supervised!({Task.Supervisor, name: Indexer.TaskSupervisor})
AddressBalanceFetcherCase.start_supervised!()
start_supervised!(Indexer.PendingTransactionFetcher)
AddressBalanceFetcherCase.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments)
start_supervised!({PendingTransactionFetcher, json_rpc_named_arguments: json_rpc_named_arguments})
wait_for_results(fn ->
Repo.one!(from(transaction in Explorer.Chain.Transaction, where: is_nil(transaction.block_hash), limit: 1))
end)
:transaction
|> insert(hash: "0x3a3eb134e6792ce9403ea4188e5e79693de9e4c94e499db132be086400da79e6")
|> with_block()
hash_strings = InternalTransactionFetcher.init([], fn hash_string, acc -> [hash_string | acc] end)
hash_strings =
InternalTransactionFetcher.init([], fn hash_string, acc -> [hash_string | acc] end, json_rpc_named_arguments)
assert :ok = InternalTransactionFetcher.run(hash_strings, 0)
assert :ok = InternalTransactionFetcher.run(hash_strings, 0, json_rpc_named_arguments)
end
describe "init/2" do
test "does not buffer pending transactions" do
test "does not buffer pending transactions", %{json_rpc_named_arguments: json_rpc_named_arguments} do
insert(:transaction)
assert InternalTransactionFetcher.init([], fn hash_string, acc -> [hash_string | acc] end) == []
assert InternalTransactionFetcher.init(
[],
fn hash_string, acc -> [hash_string | acc] end,
json_rpc_named_arguments
) == []
end
test "buffers collated transactions with unfetched internal transactions" do
test "buffers collated transactions with unfetched internal transactions", %{
json_rpc_named_arguments: json_rpc_named_arguments
} do
block = insert(:block)
collated_unfetched_transaction =
@ -41,24 +96,40 @@ defmodule Indexer.InternalTransactionFetcherTest do
|> insert()
|> with_block(block)
assert InternalTransactionFetcher.init([], fn hash_string, acc -> [hash_string | acc] end) == [
assert InternalTransactionFetcher.init(
[],
fn hash_string, acc -> [hash_string | acc] end,
json_rpc_named_arguments
) == [
%{block_number: block.number, hash_data: to_string(collated_unfetched_transaction.hash)}
]
end
test "does not buffer collated transactions with fetched internal transactions" do
test "does not buffer collated transactions with fetched internal transactions", %{
json_rpc_named_arguments: json_rpc_named_arguments
} do
:transaction
|> insert()
|> with_block(internal_transactions_indexed_at: DateTime.utc_now())
assert InternalTransactionFetcher.init([], fn hash_string, acc -> [hash_string | acc] end) == []
assert InternalTransactionFetcher.init(
[],
fn hash_string, acc -> [hash_string | acc] end,
json_rpc_named_arguments
) == []
end
end
describe "run/2" do
test "duplicate transaction hashes are logged" do
test "duplicate transaction hashes are logged", %{json_rpc_named_arguments: json_rpc_named_arguments} do
if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do
expect(EthereumJSONRPC.Mox, :json_rpc, fn _json, _options ->
{:ok, [%{id: 0, result: %{"trace" => []}}]}
end)
end
start_supervised!({Task.Supervisor, name: Indexer.TaskSupervisor})
AddressBalanceFetcherCase.start_supervised!()
AddressBalanceFetcherCase.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments)
insert(:transaction, hash: "0x03cd5899a63b6f6222afda8705d059fd5a7d126bcabe962fb654d9736e6bcafa")
@ -69,7 +140,8 @@ defmodule Indexer.InternalTransactionFetcherTest do
%{block_number: 1, hash_data: "0x03cd5899a63b6f6222afda8705d059fd5a7d126bcabe962fb654d9736e6bcafa"},
%{block_number: 1, hash_data: "0x03cd5899a63b6f6222afda8705d059fd5a7d126bcabe962fb654d9736e6bcafa"}
],
0
0,
json_rpc_named_arguments
)
end)
@ -81,10 +153,15 @@ defmodule Indexer.InternalTransactionFetcherTest do
"""
end
@tag :no_geth
test "duplicate transaction hashes only retry uniques" do
test "duplicate transaction hashes only retry uniques", %{json_rpc_named_arguments: json_rpc_named_arguments} do
if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do
expect(EthereumJSONRPC.Mox, :json_rpc, fn _json, _options ->
{:ok, [%{id: 0, error: %{code: -32602, message: "Invalid params"}}]}
end)
end
start_supervised!({Task.Supervisor, name: Indexer.TaskSupervisor})
AddressBalanceFetcherCase.start_supervised!()
AddressBalanceFetcherCase.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments)
# not a real transaction hash, so that it fails
insert(:transaction, hash: "0x0000000000000000000000000000000000000000000000000000000000000001")
@ -94,7 +171,8 @@ defmodule Indexer.InternalTransactionFetcherTest do
%{block_number: 1, hash_data: "0x0000000000000000000000000000000000000000000000000000000000000001"},
%{block_number: 1, hash_data: "0x0000000000000000000000000000000000000000000000000000000000000001"}
],
0
0,
json_rpc_named_arguments
) ==
{:retry,
[%{block_number: 1, hash_data: "0x0000000000000000000000000000000000000000000000000000000000000001"}]}

@ -1,18 +1,65 @@
defmodule Indexer.PendingTransactionFetcherTest do
# `async: false` due to use of named GenServer
use Explorer.DataCase, async: false
use EthereumJSONRPC.Case, async: false
use Explorer.DataCase
import Mox
describe "start_link/1" do
@tag :no_geth
# this test may fail if Sokol so low volume that the pending transactions are empty for too long
test "starts fetching pending transactions" do
alias Explorer.Chain.Transaction
alias Indexer.PendingTransactionFetcher
# MUST use global mode because we aren't guaranteed to get PendingTransactionFetcher's pid back fast enough to `allow`
# it to use expectations and stubs from test's pid.
setup :set_mox_global
setup :verify_on_exit!
@moduletag [capture_log: true, no_geth: true]
describe "start_link/1" do
# this test may fail if Sokol so low volume that the pending transactions are empty for too long
test "starts fetching pending transactions", %{json_rpc_named_arguments: json_rpc_named_arguments} do
if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do
EthereumJSONRPC.Mox
|> expect(:json_rpc, fn _json, _options ->
{:ok,
[
%{
"blockHash" => nil,
"blockNumber" => nil,
"chainId" => "0x4d",
"condition" => nil,
"creates" => "0xffc87239eb0267bc3ca2cd51d12fbf278e02ccb4",
"from" => "0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca",
"gas" => "0x47b760",
"gasPrice" => "0x174876e800",
"hash" => "0x3a3eb134e6792ce9403ea4188e5e79693de9e4c94e499db132be086400da79e6",
"input" =>
"0x6060604052341561000f57600080fd5b336000806101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff1602179055506102db8061005e6000396000f300606060405260043610610062576000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff1680630900f01014610067578063445df0ac146100a05780638da5cb5b146100c9578063fdacd5761461011e575b600080fd5b341561007257600080fd5b61009e600480803573ffffffffffffffffffffffffffffffffffffffff16906020019091905050610141565b005b34156100ab57600080fd5b6100b3610224565b6040518082815260200191505060405180910390f35b34156100d457600080fd5b6100dc61022a565b604051808273ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16815260200191505060405180910390f35b341561012957600080fd5b61013f600480803590602001909190505061024f565b005b60008060009054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff161415610220578190508073ffffffffffffffffffffffffffffffffffffffff1663fdacd5766001546040518263ffffffff167c010000000000000000000000000000000000000000000000000000000002815260040180828152602001915050600060405180830381600087803b151561020b57600080fd5b6102c65a03f1151561021c57600080fd5b5050505b5050565b60015481565b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1681565b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff1614156102ac57806001819055505b505600a165627a7a72305820a9c628775efbfbc17477a472413c01ee9b33881f550c59d21bee9928835c854b0029",
"nonce" => "0x0",
"publicKey" =>
"0xe5d196ad4ceada719d9e592f7166d0c75700f6eab2e3c3de34ba751ea786527cb3f6eb96ad9fdfdb9989ff572df50f1c42ef800af9c5207a38b929aff969b5c9",
"r" => "0xad3733df250c87556335ffe46c23e34dbaffde93097ef92f52c88632a40f0c75",
"raw" =>
"0xf9038d8085174876e8008347b7608080b903396060604052341561000f57600080fd5b336000806101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff1602179055506102db8061005e6000396000f300606060405260043610610062576000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff1680630900f01014610067578063445df0ac146100a05780638da5cb5b146100c9578063fdacd5761461011e575b600080fd5b341561007257600080fd5b61009e600480803573ffffffffffffffffffffffffffffffffffffffff16906020019091905050610141565b005b34156100ab57600080fd5b6100b3610224565b6040518082815260200191505060405180910390f35b34156100d457600080fd5b6100dc61022a565b604051808273ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16815260200191505060405180910390f35b341561012957600080fd5b61013f600480803590602001909190505061024f565b005b60008060009054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff161415610220578190508073ffffffffffffffffffffffffffffffffffffffff1663fdacd5766001546040518263ffffffff167c010000000000000000000000000000000000000000000000000000000002815260040180828152602001915050600060405180830381600087803b151561020b57600080fd5b6102c65a03f1151561021c57600080fd5b5050505b5050565b60015481565b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1681565b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff1614156102ac57806001819055505b505600a165627a7a72305820a9c628775efbfbc17477a472413c01ee9b33881f550c59d21bee9928835c854b002981bda0ad3733df250c87556335ffe46c23e34dbaffde93097ef92f52c88632a40f0c75a072caddc0371451a58de2ca6ab64e0f586ccdb9465ff54e1c82564940e89291e3",
"s" => "0x72caddc0371451a58de2ca6ab64e0f586ccdb9465ff54e1c82564940e89291e3",
"standardV" => "0x0",
"to" => nil,
"transactionIndex" => nil,
"v" => "0xbd",
"value" => "0x0"
}
]}
end)
|> stub(:json_rpc, fn _json, _options ->
{:ok, []}
end)
end
assert Repo.aggregate(Transaction, :count, :hash) == 0
start_supervised!({Task.Supervisor, name: Indexer.TaskSupervisor})
start_supervised!(PendingTransactionFetcher)
start_supervised!({PendingTransactionFetcher, json_rpc_named_arguments: json_rpc_named_arguments})
wait_for_results(fn ->
Repo.one!(from(transaction in Transaction, where: is_nil(transaction.block_hash), limit: 1))

@ -14,6 +14,8 @@ end
# no declared in :applications since it is test-only
{:ok, _} = Application.ensure_all_started(:ex_machina)
Mox.defmock(EthereumJSONRPC.Mox, for: EthereumJSONRPC.Transport)
ExUnit.configure(formatters: [JUnitFormatter, ExUnit.CLIFormatter])
ExUnit.start()

Loading…
Cancel
Save