CircleCI tests for Geth

pull/335/head
Luke Imhoff 7 years ago
parent c9e6366cb8
commit 4ecfb3b409
  1. 50
      .circleci/config.yml
  2. 2
      apps/ethereum_jsonrpc/config/geth.exs
  3. 62
      apps/ethereum_jsonrpc/lib/ethereum_jsonrpc.ex
  4. 59
      apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/decode_error.ex
  5. 34
      apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/geth.ex
  6. 2
      apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/parity.ex
  7. 21
      apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/variant.ex
  8. 235
      apps/ethereum_jsonrpc/test/etheream_jsonrpc_test.exs
  9. 7
      apps/ethereum_jsonrpc/test/ethereum_jsonrpc/geth_test.exs
  10. 204
      apps/ethereum_jsonrpc/test/ethereum_jsonrpc/parity_test.exs
  11. 3
      apps/indexer/lib/indexer/internal_transaction_fetcher.ex
  12. 30
      apps/indexer/lib/indexer/pending_transaction_fetcher.ex
  13. 123
      apps/indexer/test/indexer/address_balance_fetcher_test.exs
  14. 278
      apps/indexer/test/indexer/block_fetcher_test.exs
  15. 63
      apps/indexer/test/indexer/internal_transaction_fetcher_test.exs
  16. 26
      apps/indexer/test/indexer/pending_transaction_fetcher_test.exs

@ -296,7 +296,7 @@ jobs:
name: Scan explorer_web for vulnerabilities
command: mix sobelow --config
working_directory: "apps/explorer_web"
test:
test_geth:
docker:
# Ensure .tool-versions matches
- image: circleci/elixir:1.6.5-node-browsers
@ -306,6 +306,7 @@ jobs:
PGPASSWORD: postgres
# match POSTGRES_USER for postgres image below
PGUSER: postgres
ETHEREUM_JSONRPC_VARIANT: geth
- image: circleci/postgres:10.3-alpine
environment:
# Match apps/explorer/config/test.exs config :explorerer, Explorer.Repo, database
@ -328,7 +329,44 @@ jobs:
name: Wait for DB
command: dockerize -wait tcp://localhost:5432 -timeout 1m
- run: mix coveralls.circle --umbrella
- run: mix coveralls.circle --parallel --umbrella
- store_test_results:
path: _build/test/junit
test_parity:
docker:
# Ensure .tool-versions matches
- image: circleci/elixir:1.6.5-node-browsers
environment:
MIX_ENV: test
# match POSTGRES_PASSWORD for postgres image below
PGPASSWORD: postgres
# match POSTGRES_USER for postgres image below
PGUSER: postgres
ETHEREUM_JSONRPC_VARIANT: parity
- image: circleci/postgres:10.3-alpine
environment:
# Match apps/explorer/config/test.exs config :explorerer, Explorer.Repo, database
POSTGRES_DB: explorer_test
# match PGPASSWORD for elixir image above
POSTGRES_PASSWORD: postgres
# match PGUSER for elixir image above
POSTGRES_USER: postgres
working_directory: ~/app
steps:
- attach_workspace:
at: .
- run: mix local.hex --force
- run: mix local.rebar --force
- run:
name: Wait for DB
command: dockerize -wait tcp://localhost:5432 -timeout 1m
- run: mix coveralls.circle --parallel --umbrella
- store_test_results:
path: _build/test/junit
@ -356,7 +394,8 @@ workflows:
- eslint
- jest
- sobelow
- test
- test_parity
- test_geth
- dialyzer:
requires:
- build
@ -372,6 +411,9 @@ workflows:
- sobelow:
requires:
- build
- test:
- test_parity:
requires:
- build
- test_geth:
requires:
- build

@ -1,5 +1,5 @@
use Mix.Config
config :ethereum_jsonrpc,
url: "https://mainnet.infura.io/mew",
url: "https://mainnet.infura.io/8lTvJTKmHPCHazkneJsY",
variant: EthereumJSONRPC.Geth

@ -163,11 +163,6 @@ defmodule EthereumJSONRPC do
@doc """
Fetches block number by `t:tag/0`.
The `"earliest"` tag is the earlist block number, which is `0`.
iex> EthereumJSONRPC.fetch_block_number_by_tag("earliest")
{:ok, 0}
## Returns
* `{:ok, number}` - the block number for the given `tag`.
@ -184,12 +179,19 @@ defmodule EthereumJSONRPC do
end
@doc """
Fetches internal transactions from client-specific API.
Fetches internal transactions from variant API.
"""
def fetch_internal_transactions(params_list) when is_list(params_list) do
config(:variant).fetch_internal_transactions(params_list)
end
@doc """
Fetches pending transactions from variant API.
"""
def fetch_pending_transactions do
config(:variant).fetch_pending_transactions()
end
@spec fetch_transaction_receipts([
%{required(:gas) => non_neg_integer(), required(:hash) => hash, optional(atom) => any}
]) :: {:ok, %{logs: list(), receipts: list()}} | {:error, reason :: term}
@ -224,8 +226,13 @@ defmodule EthereumJSONRPC do
json = encode_json(payload)
case post(url, json, config(:http)) do
{:ok, %HTTPoison.Response{body: body, status_code: code}} ->
body |> decode_json(code, json, url) |> handle_response(code)
{:ok, %HTTPoison.Response{body: body, status_code: status_code}} ->
[
request: [url: url, body: json],
response: [status_code: status_code, body: body]
]
|> decode_json()
|> handle_response(status_code)
{:error, %HTTPoison.Error{reason: reason}} ->
{:error, reason}
@ -296,7 +303,7 @@ defmodule EthereumJSONRPC do
rechunk_json_rpc(url, chunks, options, response, decoded_response_bodies)
{:ok, %HTTPoison.Response{body: body, status_code: status_code}} ->
decoded_body = decode_json(body, status_code, json, url)
decoded_body = decode_json(request: [url: url, body: json], response: [status_code: status_code, body: body])
chunked_json_rpc(url, tail, options, [decoded_body | decoded_response_bodies])
{:error, %HTTPoison.Error{reason: reason}} ->
@ -404,7 +411,7 @@ defmodule EthereumJSONRPC do
defp get_block_by_tag_request(tag) do
# eth_getBlockByNumber accepts either a number OR a tag
get_block_by_number_request(%{id: tag, tag: tag, transactions: :hashes})
get_block_by_number_request(%{id: 1, tag: tag, transactions: :hashes})
end
defp get_block_by_number_params(options) do
@ -436,29 +443,18 @@ defmodule EthereumJSONRPC do
defp encode_json(data), do: Jason.encode_to_iodata!(data)
defp decode_json(response_body, response_status_code, request_body, request_url) do
Jason.decode!(response_body)
rescue
Jason.DecodeError ->
Logger.error(fn ->
"""
failed to decode json payload:
request:
url: #{inspect(request_url)}
body: #{inspect(request_body)}
response:
status code: #{inspect(response_status_code)}
body: #{inspect(response_body)}
"""
end)
raise("bad jason")
defp decode_json(named_arguments) when is_list(named_arguments) do
response_body =
named_arguments
|> Keyword.fetch!(:response)
|> Keyword.fetch!(:body)
try do
Jason.decode!(response_body)
rescue
Jason.DecodeError ->
raise EthereumJSONRPC.DecodeError, named_arguments
end
end
defp handle_get_blocks({:ok, results}) do

@ -0,0 +1,59 @@
defmodule EthereumJSONRPC.DecodeError do
@moduledoc """
An error has occurred decoding the response to an `EthereumJSONRPC.json_rpc` request.
"""
@enforce_keys [:request, :response]
defexception [:request, :response]
defmodule Request do
@moduledoc """
Ethereum JSONRPC request whose `EthererumJSONRPC.DecodeError.Response` had a decode error.
"""
@enforce_keys [:url, :body]
defstruct [:url, :body]
end
defmodule Response do
@moduledoc """
Ethereum JSONRPC response that had a decode error.
"""
@enforce_keys [:status_code, :body]
defstruct [:status_code, :body]
end
@impl Exception
def exception(named_arguments) do
request_fields = Keyword.fetch!(named_arguments, :request)
request = struct!(EthereumJSONRPC.DecodeError.Request, request_fields)
response_fields = Keyword.fetch!(named_arguments, :response)
response = struct!(EthereumJSONRPC.DecodeError.Response, response_fields)
%EthereumJSONRPC.DecodeError{request: request, response: response}
end
@impl Exception
def message(%EthereumJSONRPC.DecodeError{
request: %EthereumJSONRPC.DecodeError.Request{url: request_url, body: request_body},
response: %EthereumJSONRPC.DecodeError.Response{status_code: response_status_code, body: response_body}
}) do
"""
Failed to decode Ethereum JSONRPC response:
request:
url: #{request_url}
body: #{IO.iodata_to_binary(request_body)}
response:
status code: #{response_status_code}
body: #{response_body}
"""
end
end

@ -0,0 +1,34 @@
defmodule EthereumJSONRPC.Geth do
@moduledoc """
Ethereum JSONRPC methods that are only supported by [Geth](https://github.com/ethereum/go-ethereum/wiki/geth).
"""
@behaviour EthereumJSONRPC.Variant
@doc """
Internal transaction fetching is not supported currently for Geth.
To signal to the caller that fetching is not supported, `:ignore` is returned
iex> EthereumJSONRPC.Geth.fetch_internal_transactions([
...> "0x2ec382949ba0b22443aa4cb38267b1fb5e68e188109ac11f7a82f67571a0adf3"
...> ])
:ignore
"""
@impl EthereumJSONRPC.Variant
def fetch_internal_transactions(transaction_params) when is_list(transaction_params),
do: :ignore
@doc """
Pending transaction fetching is not supported currently for Geth.
To signal to the caller that fetching is not supported, `:ignore` is returned
iex> EthereumJSONRPC.Geth.fetch_pending_transactions()
:ignore
"""
@impl EthereumJSONRPC.Variant
def fetch_pending_transactions, do: :ignore
end

@ -13,6 +13,7 @@ defmodule EthereumJSONRPC.Parity do
@doc """
Fetches the `t:Explorer.Chain.InternalTransaction.changeset/2` params from the Parity trace URL.
"""
@impl EthereumJSONRPC.Variant
def fetch_internal_transactions(transactions_params) when is_list(transactions_params) do
id_to_params = id_to_params(transactions_params)
@ -30,6 +31,7 @@ defmodule EthereumJSONRPC.Parity do
*NOTE*: The pending transactions are local to the node that is contacted and may not be consistent across nodes based
on the transactions that each node has seen and how each node prioritizes collating transactions into the next block.
"""
@impl EthereumJSONRPC.Variant
@spec fetch_pending_transactions() :: {:ok, [Transaction.params()]} | {:error, reason :: term}
def fetch_pending_transactions do
with {:ok, transactions} <-

@ -10,7 +10,26 @@ defmodule EthereumJSONRPC.Variant do
@doc """
Fetches the `t:Explorer.Chain.InternalTransaction.changeset/2` params from the variant of the Ethereum JSONRPC API.
## Returns
* `{:ok, [internal_transaction_params]}` - internal transactions were successfully fetched for all transactions
* `{:error, reason}` - there was one or more errors with `reason` in fetching at least one of the transaction's
internal transactions
* `:ignore` - the variant does not support fetching internal transactions.
"""
@callback fetch_internal_transactions([Transaction.params()]) ::
{:ok, [internal_transaction_params]} | {:error, reason :: term}
{:ok, [internal_transaction_params]} | {:error, reason :: term} | :ignore
@doc """
Fetch the `t:Explorer.Chain.Transaction.changeset/2` params for pending transactions from the variant of the Ethereum
JSONRPC API.
## Returns
* `{:ok, [transaction_params]}` - pending transactions were succucessfully fetched
* `{:error, reason}` - there was one or more errors with `reason` in fetching the pending transactions
* `:ignore` - the variant does not support fetching pending transactions.
"""
@callback fetch_pending_transactions() :: {:ok, [Transaction.params()]} | {:error, reason :: term} | :ignore
end

@ -1,88 +1,166 @@
defmodule EthereumJSONRPCTest do
use ExUnit.Case, async: true
doctest EthereumJSONRPC
@moduletag :capture_log
setup do
%{variant: EthereumJSONRPC.config(:variant)}
end
describe "fetch_balances/1" do
test "with all valid hash_data returns {:ok, addresses_params}" do
assert EthereumJSONRPC.fetch_balances([
%{block_quantity: "0x1", hash_data: "0x8bf38d4764929064f2d4d3a56520a76ab3df415b"}
]) ==
{:ok,
[
%{
fetched_balance: 1,
fetched_balance_block_number: 1,
hash: "0x8bf38d4764929064f2d4d3a56520a76ab3df415b"
}
]}
test "with all valid hash_data returns {:ok, addresses_params}", %{variant: variant} do
assert {:ok,
[
%{
fetched_balance: fetched_balance,
fetched_balance_block_number: 1,
hash: "0x8bf38d4764929064f2d4d3a56520a76ab3df415b"
}
]} =
EthereumJSONRPC.fetch_balances([
%{block_quantity: "0x1", hash_data: "0x8bf38d4764929064f2d4d3a56520a76ab3df415b"}
])
case variant do
EthereumJSONRPC.Geth ->
assert fetched_balance == 0
EthereumJSONRPC.Parity ->
assert fetched_balance == 1
_ ->
raise ArgumentError, "Unsupported variant (#{variant}})"
end
end
test "with all invalid hash_data returns {:error, reasons}" do
assert EthereumJSONRPC.fetch_balances([%{block_quantity: "0x1", hash_data: "0x0"}]) ==
{:error,
[
%{
"blockNumber" => "0x1",
"code" => -32602,
"hash" => "0x0",
"message" =>
"Invalid params: invalid length 1, expected a 0x-prefixed, padded, hex-encoded hash with length 40."
}
]}
test "with all invalid hash_data returns {:error, reasons}", %{variant: variant} do
assert {:error, reasons} = EthereumJSONRPC.fetch_balances([%{block_quantity: "0x1", hash_data: "0x0"}])
assert is_list(reasons)
assert length(reasons) == 1
[reason] = reasons
assert %{
"blockNumber" => "0x1",
"code" => -32602,
"hash" => "0x0",
"message" => message
} = reason
case variant do
EthereumJSONRPC.Geth ->
assert message ==
"invalid argument 0: json: cannot unmarshal hex string of odd length into Go value of type common.Address"
EthereumJSONRPC.Parity ->
assert message ==
"Invalid params: invalid length 1, expected a 0x-prefixed, padded, hex-encoded hash with length 40."
_ ->
raise ArgumentError, "Unsupported variant (#{variant}})"
end
end
test "with a mix of valid and invalid hash_data returns {:error, reasons}" do
assert EthereumJSONRPC.fetch_balances([
# start with :ok
%{
block_quantity: "0x1",
hash_data: "0x8bf38d4764929064f2d4d3a56520a76ab3df415b"
},
# :ok, :ok clause
%{
block_quantity: "0x34",
hash_data: "0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca"
},
# :ok, :error clause
%{
block_quantity: "0x2",
hash_data: "0x3"
},
# :error, :ok clause
%{
block_quantity: "0x35",
hash_data: "0x8bf38d4764929064f2d4d3a56520a76ab3df415b"
},
# :error, :error clause
%{
block_quantity: "0x4",
hash_data: "0x5"
}
]) ==
{:error,
[
%{
"blockNumber" => "0x2",
"code" => -32602,
"hash" => "0x3",
"message" =>
"Invalid params: invalid length 1, expected a 0x-prefixed, padded, hex-encoded hash with length 40."
},
%{
"blockNumber" => "0x4",
"code" => -32602,
"hash" => "0x5",
"message" =>
"Invalid params: invalid length 1, expected a 0x-prefixed, padded, hex-encoded hash with length 40."
}
]}
test "with a mix of valid and invalid hash_data returns {:error, reasons}", %{variant: variant} do
assert {:error, reasons} =
EthereumJSONRPC.fetch_balances([
# start with :ok
%{
block_quantity: "0x1",
hash_data: "0x8bf38d4764929064f2d4d3a56520a76ab3df415b"
},
# :ok, :ok clause
%{
block_quantity: "0x34",
hash_data: "0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca"
},
# :ok, :error clause
%{
block_quantity: "0x2",
hash_data: "0x3"
},
# :error, :ok clause
%{
block_quantity: "0x35",
hash_data: "0x8bf38d4764929064f2d4d3a56520a76ab3df415b"
},
# :error, :error clause
%{
block_quantity: "0x4",
hash_data: "0x5"
}
])
assert is_list(reasons)
assert length(reasons) == 2
reason_by_hash_by_block_number =
Enum.reduce(reasons, %{}, fn %{"blockNumber" => block_number, "hash" => hash} = reason, acc ->
put_in(acc, [Access.key(hash, %{}), Access.key(block_number)], reason)
end)
case variant do
EthereumJSONRPC.Geth ->
assert reason_by_hash_by_block_number["0x3"]["0x2"] == %{
"blockNumber" => "0x2",
"code" => -32602,
"hash" => "0x3",
"message" =>
"invalid argument 0: json: cannot unmarshal hex string of odd length into Go value of type common.Address"
}
assert reason_by_hash_by_block_number["0x5"]["0x4"] == %{
"blockNumber" => "0x4",
"code" => -32602,
"hash" => "0x5",
"message" =>
"invalid argument 0: json: cannot unmarshal hex string of odd length into Go value of type common.Address"
}
EthereumJSONRPC.Parity ->
assert reason_by_hash_by_block_number["0x3"]["0x2"] == %{
"blockNumber" => "0x2",
"code" => -32602,
"hash" => "0x3",
"message" =>
"Invalid params: invalid length 1, expected a 0x-prefixed, padded, hex-encoded hash with length 40."
}
assert reason_by_hash_by_block_number["0x5"]["0x4"] == %{
"blockNumber" => "0x4",
"code" => -32602,
"hash" => "0x5",
"message" =>
"Invalid params: invalid length 1, expected a 0x-prefixed, padded, hex-encoded hash with length 40."
}
_ ->
raise ArgumentError, "Unsupported variant (#{variant})"
end
end
end
describe "fetch_block_number_by_tag/1" do
test "with earliest" do
assert {:ok, 0} = EthereumJSONRPC.fetch_block_number_by_tag("earliest")
end
test "with latest" do
assert {:ok, number} = EthereumJSONRPC.fetch_block_number_by_tag("latest")
assert number > 0
end
test "with pending" do
assert {:ok, number} = EthereumJSONRPC.fetch_block_number_by_tag("pending")
assert number > 0
end
end
describe "json_rpc/2" do
# regression test for https://github.com/poanetwork/poa-explorer/issues/254
test "transparently splits batch payloads that would trigger a 413 Request Entity Too Large" do
test "transparently splits batch payloads that would trigger a 413 Request Entity Too Large", %{variant: variant} do
block_numbers = 0..13000
payload =
@ -90,7 +168,7 @@ defmodule EthereumJSONRPCTest do
|> Stream.with_index()
|> Enum.map(&get_block_by_number_request/1)
assert_payload_too_large(payload)
assert_payload_too_large(payload, variant)
url = EthereumJSONRPC.config(:url)
@ -108,7 +186,7 @@ defmodule EthereumJSONRPCTest do
end
end
defp assert_payload_too_large(payload) do
defp assert_payload_too_large(payload, variant) do
json = Jason.encode_to_iodata!(payload)
headers = [{"Content-Type", "application/json"}]
url = EthereumJSONRPC.config(:url)
@ -116,7 +194,16 @@ defmodule EthereumJSONRPCTest do
assert {:ok, %HTTPoison.Response{body: body, status_code: 413}} =
HTTPoison.post(url, json, headers, EthereumJSONRPC.config(:http))
assert body =~ "413 Request Entity Too Large"
case variant do
EthereumJSONRPC.Geth ->
assert body =~ "content length too large"
EthereumJSONRPC.Parity ->
assert body =~ "413 Request Entity Too Large"
_ ->
raise ArgumentError, "Unsupported variant (#{variant})"
end
end
defp get_block_by_number_request({block_number, id}) do

@ -0,0 +1,7 @@
defmodule EthereumJSONRPC.GethTest do
use ExUnit.Case, async: false
if EthereumJSONRPC.config(:variant) == EthereumJSONRPC.Geth do
doctest EthereumJSONRPC.Geth
end
end

@ -1,113 +1,115 @@
defmodule EthereumJSONRPC.ParityTest do
use ExUnit.Case, async: true
doctest EthereumJSONRPC.Parity
if EthereumJSONRPC.config(:variant) == EthereumJSONRPC.Parity do
doctest EthereumJSONRPC.Parity
describe "fetch_internal_transactions/1" do
test "with all valid transaction_params returns {:ok, transactions_params}" do
assert EthereumJSONRPC.Parity.fetch_internal_transactions([
%{
block_number: 1,
hash_data: "0x0fa6f723216dba694337f9bb37d8870725655bdf2573526a39454685659e39b1"
}
]) == {
:ok,
[
describe "fetch_internal_transactions/1" do
test "with all valid transaction_params returns {:ok, transactions_params}" do
assert EthereumJSONRPC.Parity.fetch_internal_transactions([
%{
block_number: 1,
created_contract_address_hash: "0x1e0eaa06d02f965be2dfe0bc9ff52b2d82133461",
created_contract_code:
"0x60606040526004361061008e576000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff168063247b3210146100935780632ffdfc8a146100bc57806374294144146100f6578063ae4b1b5b14610125578063bf7370d11461017a578063d1104cb2146101a3578063eecd1079146101f8578063fcff021c14610221575b600080fd5b341561009e57600080fd5b6100a661024a565b6040518082815260200191505060405180910390f35b34156100c757600080fd5b6100e0600480803560ff16906020019091905050610253565b6040518082815260200191505060405180910390f35b341561010157600080fd5b610123600480803590602001909190803560ff16906020019091905050610276565b005b341561013057600080fd5b61013861037a565b604051808273ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16815260200191505060405180910390f35b341561018557600080fd5b61018d61039f565b6040518082815260200191505060405180910390f35b34156101ae57600080fd5b6101b66104d9565b604051808273ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16815260200191505060405180910390f35b341561020357600080fd5b61020b610588565b6040518082815260200191505060405180910390f35b341561022c57600080fd5b6102346105bd565b6040518082815260200191505060405180910390f35b600060c8905090565b6000600160008360ff1660ff168152602001908152602001600020549050919050565b61027e6104d9565b73ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff161415156102b757600080fd5b60008160ff161115156102c957600080fd5b6002808111156102d557fe5b60ff168160ff16111515156102e957600080fd5b6000821180156103125750600160008260ff1660ff168152602001908152602001600020548214155b151561031d57600080fd5b81600160008360ff1660ff168152602001908152602001600020819055508060ff167fe868bbbdd6cd2efcd9ba6e0129d43c349b0645524aba13f8a43bfc7c5ffb0889836040518082815260200191505060405180910390a25050565b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1681565b6000806000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16638b8414c46000604051602001526040518163ffffffff167c0100000000000000000000000000000000000000000000000000000000028152600401602060405180830381600087803b151561042f57600080fd5b6102c65a03f1151561044057600080fd5b5050506040518051905090508073ffffffffffffffffffffffffffffffffffffffff16630eaba26a6000604051602001526040518163ffffffff167c0100000000000000000000000000000000000000000000000000000000028152600401602060405180830381600087803b15156104b857600080fd5b6102c65a03f115156104c957600080fd5b5050506040518051905091505090565b60008060009054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff1663a3b3fff16000604051602001526040518163ffffffff167c0100000000000000000000000000000000000000000000000000000000028152600401602060405180830381600087803b151561056857600080fd5b6102c65a03f1151561057957600080fd5b50505060405180519050905090565b60006105b860016105aa600261059c61039f565b6105e590919063ffffffff16565b61060090919063ffffffff16565b905090565b60006105e06105ca61039f565b6105d261024a565b6105e590919063ffffffff16565b905090565b60008082848115156105f357fe5b0490508091505092915050565b600080828401905083811015151561061457fe5b80915050929150505600a165627a7a723058206b7eef2a57eb659d5e77e45ab5bc074e99c6a841921038cdb931e119c6aac46c0029",
from_address_hash: "0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca",
gas: 4_533_872,
gas_used: 382_953,
index: 0,
init:
"0x6060604052341561000f57600080fd5b60405160208061071a83398101604052808051906020019091905050806000806101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff1602179055506003600160006001600281111561007e57fe5b60ff1660ff168152602001908152602001600020819055506002600160006002808111156100a857fe5b60ff1660ff168152602001908152602001600020819055505061064a806100d06000396000f30060606040526004361061008e576000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff168063247b3210146100935780632ffdfc8a146100bc57806374294144146100f6578063ae4b1b5b14610125578063bf7370d11461017a578063d1104cb2146101a3578063eecd1079146101f8578063fcff021c14610221575b600080fd5b341561009e57600080fd5b6100a661024a565b6040518082815260200191505060405180910390f35b34156100c757600080fd5b6100e0600480803560ff16906020019091905050610253565b6040518082815260200191505060405180910390f35b341561010157600080fd5b610123600480803590602001909190803560ff16906020019091905050610276565b005b341561013057600080fd5b61013861037a565b604051808273ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16815260200191505060405180910390f35b341561018557600080fd5b61018d61039f565b6040518082815260200191505060405180910390f35b34156101ae57600080fd5b6101b66104d9565b604051808273ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16815260200191505060405180910390f35b341561020357600080fd5b61020b610588565b6040518082815260200191505060405180910390f35b341561022c57600080fd5b6102346105bd565b6040518082815260200191505060405180910390f35b600060c8905090565b6000600160008360ff1660ff168152602001908152602001600020549050919050565b61027e6104d9565b73ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff161415156102b757600080fd5b60008160ff161115156102c957600080fd5b6002808111156102d557fe5b60ff168160ff16111515156102e957600080fd5b6000821180156103125750600160008260ff1660ff168152602001908152602001600020548214155b151561031d57600080fd5b81600160008360ff1660ff168152602001908152602001600020819055508060ff167fe868bbbdd6cd2efcd9ba6e0129d43c349b0645524aba13f8a43bfc7c5ffb0889836040518082815260200191505060405180910390a25050565b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1681565b6000806000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16638b8414c46000604051602001526040518163ffffffff167c0100000000000000000000000000000000000000000000000000000000028152600401602060405180830381600087803b151561042f57600080fd5b6102c65a03f1151561044057600080fd5b5050506040518051905090508073ffffffffffffffffffffffffffffffffffffffff16630eaba26a6000604051602001526040518163ffffffff167c0100000000000000000000000000000000000000000000000000000000028152600401602060405180830381600087803b15156104b857600080fd5b6102c65a03f115156104c957600080fd5b5050506040518051905091505090565b60008060009054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff1663a3b3fff16000604051602001526040518163ffffffff167c0100000000000000000000000000000000000000000000000000000000028152600401602060405180830381600087803b151561056857600080fd5b6102c65a03f1151561057957600080fd5b50505060405180519050905090565b60006105b860016105aa600261059c61039f565b6105e590919063ffffffff16565b61060090919063ffffffff16565b905090565b60006105e06105ca61039f565b6105d261024a565b6105e590919063ffffffff16565b905090565b60008082848115156105f357fe5b0490508091505092915050565b600080828401905083811015151561061457fe5b80915050929150505600a165627a7a723058206b7eef2a57eb659d5e77e45ab5bc074e99c6a841921038cdb931e119c6aac46c0029000000000000000000000000862d67cb0773ee3f8ce7ea89b328ffea861ab3ef",
trace_address: [],
transaction_hash: "0x0fa6f723216dba694337f9bb37d8870725655bdf2573526a39454685659e39b1",
type: "create",
value: 0
hash_data: "0x0fa6f723216dba694337f9bb37d8870725655bdf2573526a39454685659e39b1"
}
]
}
end
test "with all invalid transaction_params returns {:error, reasons}" do
assert EthereumJSONRPC.Parity.fetch_internal_transactions([
%{
block_number: 1,
hash_data: "0x0000000000000000000000000000000000000000000000000000000000000001"
]) == {
:ok,
[
%{
block_number: 1,
created_contract_address_hash: "0x1e0eaa06d02f965be2dfe0bc9ff52b2d82133461",
created_contract_code:
"0x60606040526004361061008e576000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff168063247b3210146100935780632ffdfc8a146100bc57806374294144146100f6578063ae4b1b5b14610125578063bf7370d11461017a578063d1104cb2146101a3578063eecd1079146101f8578063fcff021c14610221575b600080fd5b341561009e57600080fd5b6100a661024a565b6040518082815260200191505060405180910390f35b34156100c757600080fd5b6100e0600480803560ff16906020019091905050610253565b6040518082815260200191505060405180910390f35b341561010157600080fd5b610123600480803590602001909190803560ff16906020019091905050610276565b005b341561013057600080fd5b61013861037a565b604051808273ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16815260200191505060405180910390f35b341561018557600080fd5b61018d61039f565b6040518082815260200191505060405180910390f35b34156101ae57600080fd5b6101b66104d9565b604051808273ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16815260200191505060405180910390f35b341561020357600080fd5b61020b610588565b6040518082815260200191505060405180910390f35b341561022c57600080fd5b6102346105bd565b6040518082815260200191505060405180910390f35b600060c8905090565b6000600160008360ff1660ff168152602001908152602001600020549050919050565b61027e6104d9565b73ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff161415156102b757600080fd5b60008160ff161115156102c957600080fd5b6002808111156102d557fe5b60ff168160ff16111515156102e957600080fd5b6000821180156103125750600160008260ff1660ff168152602001908152602001600020548214155b151561031d57600080fd5b81600160008360ff1660ff168152602001908152602001600020819055508060ff167fe868bbbdd6cd2efcd9ba6e0129d43c349b0645524aba13f8a43bfc7c5ffb0889836040518082815260200191505060405180910390a25050565b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1681565b6000806000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16638b8414c46000604051602001526040518163ffffffff167c0100000000000000000000000000000000000000000000000000000000028152600401602060405180830381600087803b151561042f57600080fd5b6102c65a03f1151561044057600080fd5b5050506040518051905090508073ffffffffffffffffffffffffffffffffffffffff16630eaba26a6000604051602001526040518163ffffffff167c0100000000000000000000000000000000000000000000000000000000028152600401602060405180830381600087803b15156104b857600080fd5b6102c65a03f115156104c957600080fd5b5050506040518051905091505090565b60008060009054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff1663a3b3fff16000604051602001526040518163ffffffff167c0100000000000000000000000000000000000000000000000000000000028152600401602060405180830381600087803b151561056857600080fd5b6102c65a03f1151561057957600080fd5b50505060405180519050905090565b60006105b860016105aa600261059c61039f565b6105e590919063ffffffff16565b61060090919063ffffffff16565b905090565b60006105e06105ca61039f565b6105d261024a565b6105e590919063ffffffff16565b905090565b60008082848115156105f357fe5b0490508091505092915050565b600080828401905083811015151561061457fe5b80915050929150505600a165627a7a723058206b7eef2a57eb659d5e77e45ab5bc074e99c6a841921038cdb931e119c6aac46c0029",
from_address_hash: "0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca",
gas: 4_533_872,
gas_used: 382_953,
index: 0,
init:
"0x6060604052341561000f57600080fd5b60405160208061071a83398101604052808051906020019091905050806000806101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff1602179055506003600160006001600281111561007e57fe5b60ff1660ff168152602001908152602001600020819055506002600160006002808111156100a857fe5b60ff1660ff168152602001908152602001600020819055505061064a806100d06000396000f30060606040526004361061008e576000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff168063247b3210146100935780632ffdfc8a146100bc57806374294144146100f6578063ae4b1b5b14610125578063bf7370d11461017a578063d1104cb2146101a3578063eecd1079146101f8578063fcff021c14610221575b600080fd5b341561009e57600080fd5b6100a661024a565b6040518082815260200191505060405180910390f35b34156100c757600080fd5b6100e0600480803560ff16906020019091905050610253565b6040518082815260200191505060405180910390f35b341561010157600080fd5b610123600480803590602001909190803560ff16906020019091905050610276565b005b341561013057600080fd5b61013861037a565b604051808273ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16815260200191505060405180910390f35b341561018557600080fd5b61018d61039f565b6040518082815260200191505060405180910390f35b34156101ae57600080fd5b6101b66104d9565b604051808273ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16815260200191505060405180910390f35b341561020357600080fd5b61020b610588565b6040518082815260200191505060405180910390f35b341561022c57600080fd5b6102346105bd565b6040518082815260200191505060405180910390f35b600060c8905090565b6000600160008360ff1660ff168152602001908152602001600020549050919050565b61027e6104d9565b73ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff161415156102b757600080fd5b60008160ff161115156102c957600080fd5b6002808111156102d557fe5b60ff168160ff16111515156102e957600080fd5b6000821180156103125750600160008260ff1660ff168152602001908152602001600020548214155b151561031d57600080fd5b81600160008360ff1660ff168152602001908152602001600020819055508060ff167fe868bbbdd6cd2efcd9ba6e0129d43c349b0645524aba13f8a43bfc7c5ffb0889836040518082815260200191505060405180910390a25050565b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1681565b6000806000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16638b8414c46000604051602001526040518163ffffffff167c0100000000000000000000000000000000000000000000000000000000028152600401602060405180830381600087803b151561042f57600080fd5b6102c65a03f1151561044057600080fd5b5050506040518051905090508073ffffffffffffffffffffffffffffffffffffffff16630eaba26a6000604051602001526040518163ffffffff167c0100000000000000000000000000000000000000000000000000000000028152600401602060405180830381600087803b15156104b857600080fd5b6102c65a03f115156104c957600080fd5b5050506040518051905091505090565b60008060009054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff1663a3b3fff16000604051602001526040518163ffffffff167c0100000000000000000000000000000000000000000000000000000000028152600401602060405180830381600087803b151561056857600080fd5b6102c65a03f1151561057957600080fd5b50505060405180519050905090565b60006105b860016105aa600261059c61039f565b6105e590919063ffffffff16565b61060090919063ffffffff16565b905090565b60006105e06105ca61039f565b6105d261024a565b6105e590919063ffffffff16565b905090565b60008082848115156105f357fe5b0490508091505092915050565b600080828401905083811015151561061457fe5b80915050929150505600a165627a7a723058206b7eef2a57eb659d5e77e45ab5bc074e99c6a841921038cdb931e119c6aac46c0029000000000000000000000000862d67cb0773ee3f8ce7ea89b328ffea861ab3ef",
trace_address: [],
transaction_hash: "0x0fa6f723216dba694337f9bb37d8870725655bdf2573526a39454685659e39b1",
type: "create",
value: 0
}
]
}
]) ==
{:error,
[
%{
"blockNumber" => 1,
"code" => -32603,
"data" => "TransactionNotFound",
"message" =>
"Internal error occurred: {}, this should not be the case with eth_call, most likely a bug.",
"transactionHash" => "0x0000000000000000000000000000000000000000000000000000000000000001"
}
]}
end
end
test "with a mix of valid and invalid transaction_params returns {:error, reasons}" do
assert EthereumJSONRPC.Parity.fetch_internal_transactions([
# start with :ok
%{
block_number: 1,
hash_data: "0x0fa6f723216dba694337f9bb37d8870725655bdf2573526a39454685659e39b1"
},
# :ok, :ok clause
%{
block_number: 34,
hash_data: "0x3a3eb134e6792ce9403ea4188e5e79693de9e4c94e499db132be086400da79e6"
},
# :ok, :error clause
%{
block_number: 1,
hash_data: "0x0000000000000000000000000000000000000000000000000000000000000001"
},
# :error, :ok clause
%{
block_number: 35,
hash_data: "0x6b80a90c958fb5791a070929379ed6eb7a33ecdf9f9cafcada2f6803b3f25ec3"
},
# :error, :error clause
%{
block_number: 2,
hash_data: "0x0000000000000000000000000000000000000000000000000000000000000002"
}
]) ==
{:error,
[
%{
"blockNumber" => 1,
"code" => -32603,
"data" => "TransactionNotFound",
"message" =>
"Internal error occurred: {}, this should not be the case with eth_call, most likely a bug.",
"transactionHash" => "0x0000000000000000000000000000000000000000000000000000000000000001"
},
%{
"blockNumber" => 35,
"code" => -32603,
"data" => "TransactionNotFound",
"message" =>
"Internal error occurred: {}, this should not be the case with eth_call, most likely a bug.",
"transactionHash" => "0x6b80a90c958fb5791a070929379ed6eb7a33ecdf9f9cafcada2f6803b3f25ec3"
},
%{
"blockNumber" => 2,
"code" => -32603,
"data" => "TransactionNotFound",
"message" =>
"Internal error occurred: {}, this should not be the case with eth_call, most likely a bug.",
"transactionHash" => "0x0000000000000000000000000000000000000000000000000000000000000002"
}
]}
test "with all invalid transaction_params returns {:error, reasons}" do
assert EthereumJSONRPC.Parity.fetch_internal_transactions([
%{
block_number: 1,
hash_data: "0x0000000000000000000000000000000000000000000000000000000000000001"
}
]) ==
{:error,
[
%{
"blockNumber" => 1,
"code" => -32603,
"data" => "TransactionNotFound",
"message" =>
"Internal error occurred: {}, this should not be the case with eth_call, most likely a bug.",
"transactionHash" => "0x0000000000000000000000000000000000000000000000000000000000000001"
}
]}
end
test "with a mix of valid and invalid transaction_params returns {:error, reasons}" do
assert EthereumJSONRPC.Parity.fetch_internal_transactions([
# start with :ok
%{
block_number: 1,
hash_data: "0x0fa6f723216dba694337f9bb37d8870725655bdf2573526a39454685659e39b1"
},
# :ok, :ok clause
%{
block_number: 34,
hash_data: "0x3a3eb134e6792ce9403ea4188e5e79693de9e4c94e499db132be086400da79e6"
},
# :ok, :error clause
%{
block_number: 1,
hash_data: "0x0000000000000000000000000000000000000000000000000000000000000001"
},
# :error, :ok clause
%{
block_number: 35,
hash_data: "0x6b80a90c958fb5791a070929379ed6eb7a33ecdf9f9cafcada2f6803b3f25ec3"
},
# :error, :error clause
%{
block_number: 2,
hash_data: "0x0000000000000000000000000000000000000000000000000000000000000002"
}
]) ==
{:error,
[
%{
"blockNumber" => 1,
"code" => -32603,
"data" => "TransactionNotFound",
"message" =>
"Internal error occurred: {}, this should not be the case with eth_call, most likely a bug.",
"transactionHash" => "0x0000000000000000000000000000000000000000000000000000000000000001"
},
%{
"blockNumber" => 35,
"code" => -32603,
"data" => "TransactionNotFound",
"message" =>
"Internal error occurred: {}, this should not be the case with eth_call, most likely a bug.",
"transactionHash" => "0x6b80a90c958fb5791a070929379ed6eb7a33ecdf9f9cafcada2f6803b3f25ec3"
},
%{
"blockNumber" => 2,
"code" => -32603,
"data" => "TransactionNotFound",
"message" =>
"Internal error occurred: {}, this should not be the case with eth_call, most likely a bug.",
"transactionHash" => "0x0000000000000000000000000000000000000000000000000000000000000002"
}
]}
end
end
end
end

@ -125,6 +125,9 @@ defmodule Indexer.InternalTransactionFetcher do
# re-queue the de-duped transactions_params
{:retry, unique_transactions_params}
:ignore ->
:ok
end
end

@ -9,7 +9,7 @@ defmodule Indexer.PendingTransactionFetcher do
require Logger
import EthereumJSONRPC.Parity, only: [fetch_pending_transactions: 0]
import EthereumJSONRPC, only: [fetch_pending_transactions: 0]
alias Explorer.Chain
alias Indexer.{AddressExtraction, PendingTransactionFetcher}
@ -85,17 +85,21 @@ defmodule Indexer.PendingTransactionFetcher do
end
defp task(%PendingTransactionFetcher{} = _state) do
{:ok, transactions_params} = fetch_pending_transactions()
addresses_params = AddressExtraction.extract_addresses(%{transactions: transactions_params}, pending: true)
# There's no need to queue up fetching the address balance since theses are pending transactions and cannot have
# affected the address balance yet since address balance is a balance at a give block and these transactions are
# blockless.
{:ok, _} =
Chain.import_blocks(
addresses: [params: addresses_params],
transactions: [on_conflict: :nothing, params: transactions_params]
)
case fetch_pending_transactions() do
{:ok, transactions_params} ->
addresses_params = AddressExtraction.extract_addresses(%{transactions: transactions_params}, pending: true)
# There's no need to queue up fetching the address balance since theses are pending transactions and cannot have
# affected the address balance yet since address balance is a balance at a give block and these transactions are
# blockless.
{:ok, _} =
Chain.import_blocks(
addresses: [params: addresses_params],
transactions: [on_conflict: :nothing, params: transactions_params]
)
:ignore ->
:ok
end
end
end

@ -6,23 +6,37 @@ defmodule Indexer.AddressBalanceFetcherTest do
alias Explorer.Chain.{Address, Hash, Wei}
alias Indexer.{AddressBalanceFetcher, AddressBalanceFetcherCase}
@block_number 2_932_838
@hash %Explorer.Chain.Hash{
byte_count: 20,
bytes: <<139, 243, 141, 71, 100, 146, 144, 100, 242, 212, 211, 165, 101, 32, 167, 106, 179, 223, 65, 91>>
}
setup do
start_supervised!({Task.Supervisor, name: Indexer.TaskSupervisor})
:ok
%{variant: EthereumJSONRPC.config(:variant)}
end
describe "init/1" do
test "fetches unfetched Block miner balance" do
{:ok, miner_hash} = Hash.Address.cast("0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca")
test "fetches unfetched Block miner balance", %{variant: variant} do
%{block_number: block_number, fetched_balance: fetched_balance, miner_hash_data: miner_hash_data} =
case variant do
EthereumJSONRPC.Geth ->
%{
block_number: 201_480,
fetched_balance: 6_301_752_965_671_077_173,
miner_hash_data: "0xe6a7a1d47ff21b6321162aea7c6cb457d5476bca"
}
EthereumJSONRPC.Parity ->
%{
block_number: 34,
fetched_balance: 252_460_834_000_000_000_000_000_000,
miner_hash_data: "0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca"
}
_ ->
raise ArgumentError, "Unsupported variant (#{variant})"
end
{:ok, miner_hash} = Hash.Address.cast(miner_hash_data)
miner = insert(:address, hash: miner_hash)
block = insert(:block, miner: miner, number: 34)
block = insert(:block, miner: miner, number: block_number)
assert miner.fetched_balance == nil
assert miner.fetched_balance_block_number == nil
@ -36,14 +50,34 @@ defmodule Indexer.AddressBalanceFetcherTest do
)
end)
assert fetched_address.fetched_balance == %Wei{value: Decimal.new(252_460_834_000_000_000_000_000_000)}
assert fetched_address.fetched_balance == %Wei{value: Decimal.new(fetched_balance)}
assert fetched_address.fetched_balance_block_number == block.number
end
test "fetches unfetched addresses when less than max batch size" do
{:ok, miner_hash} = Hash.Address.cast("0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca")
test "fetches unfetched addresses when less than max batch size", %{variant: variant} do
%{block_number: block_number, fetched_balance: fetched_balance, miner_hash_data: miner_hash_data} =
case variant do
EthereumJSONRPC.Geth ->
%{
block_number: 201_480,
fetched_balance: 6_301_752_965_671_077_173,
miner_hash_data: "0xe6a7a1d47ff21b6321162aea7c6cb457d5476bca"
}
EthereumJSONRPC.Parity ->
%{
block_number: 34,
fetched_balance: 252_460_834_000_000_000_000_000_000,
miner_hash_data: "0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca"
}
_ ->
raise ArgumentError, "Unsupported variant (#{variant})"
end
{:ok, miner_hash} = Hash.Address.cast(miner_hash_data)
miner = insert(:address, hash: miner_hash)
block = insert(:block, miner: miner, number: 34)
block = insert(:block, miner: miner, number: block_number)
AddressBalanceFetcherCase.start_supervised!(max_batch_size: 2)
@ -54,30 +88,73 @@ defmodule Indexer.AddressBalanceFetcherTest do
)
end)
assert fetched_address.fetched_balance == %Wei{value: Decimal.new(252_460_834_000_000_000_000_000_000)}
assert fetched_address.fetched_balance == %Wei{value: Decimal.new(fetched_balance)}
assert fetched_address.fetched_balance_block_number == block.number
end
end
describe "async_fetch_balances/1" do
test "fetches balances for address_hashes" do
test "fetches balances for address_hashes", %{variant: variant} do
AddressBalanceFetcherCase.start_supervised!()
assert :ok = AddressBalanceFetcher.async_fetch_balances([%{block_number: @block_number, hash: @hash}])
%{block_number: block_number, fetched_balance: fetched_balance, hash: hash} =
case variant do
EthereumJSONRPC.Geth ->
%{
block_number: 201_480,
fetched_balance: 6_301_752_965_671_077_173,
hash: %Explorer.Chain.Hash{
byte_count: 20,
bytes: <<230, 167, 161, 212, 127, 242, 27, 99, 33, 22, 42, 234, 124, 108, 180, 87, 213, 71, 107, 202>>
}
}
EthereumJSONRPC.Parity ->
%{
block_number: 34,
fetched_balance: 252_460_834_000_000_000_000_000_000,
hash: %Explorer.Chain.Hash{
byte_count: 20,
bytes:
<<232, 221, 197, 199, 162, 210, 240, 215, 169, 121, 132, 89, 192, 16, 79, 223, 94, 152, 122, 202>>
}
}
_ ->
raise ArgumentError, "Unsupported variant (#{variant})"
end
assert :ok = AddressBalanceFetcher.async_fetch_balances([%{block_number: block_number, hash: hash}])
address =
wait(fn ->
Repo.get!(Address, @hash)
Repo.get!(Address, hash)
end)
assert address.fetched_balance == %Wei{value: Decimal.new(1)}
assert address.fetched_balance_block_number == @block_number
assert address.fetched_balance == %Wei{value: Decimal.new(fetched_balance)}
assert address.fetched_balance_block_number == block_number
end
end
describe "run/2" do
test "duplicate address hashes the max block_quantity" do
hash_data = "0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca"
test "duplicate address hashes the max block_quantity", %{variant: variant} do
%{fetched_balance: fetched_balance, hash_data: hash_data} =
case variant do
EthereumJSONRPC.Geth ->
%{
fetched_balance: 5_000_000_000_000_000_000,
hash_data: "0x05a56e2d52c817161883f50c441c3228cfe54d9f"
}
EthereumJSONRPC.Parity ->
%{
fetched_balance: 252_460_802_000_000_000_000_000_000,
hash_data: "0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca"
}
_ ->
raise ArgumentError, "Unsupported variant (#{variant})"
end
assert AddressBalanceFetcher.run(
[%{block_quantity: "0x1", hash_data: hash_data}, %{block_quantity: "0x2", hash_data: hash_data}],
@ -87,7 +164,7 @@ defmodule Indexer.AddressBalanceFetcherTest do
fetched_address = Repo.one!(from(address in Address, where: address.hash == ^hash_data))
assert fetched_address.fetched_balance == %Explorer.Chain.Wei{
value: Decimal.new(252_460_802_000_000_000_000_000_000)
value: Decimal.new(fetched_balance)
}
assert fetched_address.fetched_balance_block_number == 2

@ -36,6 +36,10 @@ defmodule Indexer.BlockFetcherTest do
# ON blocks.hash = transactions.block_hash) as blocks
@first_full_block_number 37
setup do
%{variant: EthereumJSONRPC.config(:variant)}
end
describe "start_link/1" do
test "starts fetching blocks from latest and goes down" do
{:ok, latest_block_number} = EthereumJSONRPC.fetch_block_number_by_tag("latest")
@ -137,25 +141,47 @@ defmodule Indexer.BlockFetcherTest do
%{state: state}
end
test "with single element range that is valid imports one block", %{state: state} do
test "with single element range that is valid imports one block", %{state: state, variant: variant} do
{:ok, sequence} = Sequence.start_link([], 0, 1)
%{address_hash: address_hash, block_hash: block_hash} =
case variant do
EthereumJSONRPC.Geth ->
%{
address_hash: %Explorer.Chain.Hash{
byte_count: 20,
bytes: <<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>>
},
block_hash: %Explorer.Chain.Hash{
byte_count: 32,
bytes:
<<212, 229, 103, 64, 248, 118, 174, 248, 192, 16, 184, 106, 64, 213, 245, 103, 69, 161, 24, 208, 144,
106, 52, 230, 154, 236, 140, 13, 177, 203, 143, 163>>
}
}
EthereumJSONRPC.Parity ->
%{
address_hash: %Explorer.Chain.Hash{
byte_count: 20,
bytes: <<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>>
},
block_hash: %Explorer.Chain.Hash{
byte_count: 32,
bytes:
<<91, 40, 193, 191, 211, 161, 82, 48, 201, 164, 107, 57, 156, 208, 249, 166, 146, 13, 67, 46, 133, 56,
28, 198, 161, 64, 176, 110, 132, 16, 17, 47>>
}
}
_ ->
raise ArgumenrError, "Unsupported variant (#{variant})"
end
assert {:ok,
%{
addresses: [
%Explorer.Chain.Hash{
byte_count: 20,
bytes: <<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>>
} = address_hash
],
blocks: [
%Explorer.Chain.Hash{
byte_count: 32,
bytes:
<<91, 40, 193, 191, 211, 161, 82, 48, 201, 164, 107, 57, 156, 208, 249, 166, 146, 13, 67, 46, 133,
56, 28, 198, 161, 64, 176, 110, 132, 16, 17, 47>>
}
],
addresses: [^address_hash],
blocks: [^block_hash],
logs: [],
transactions: []
}} = BlockFetcher.import_range(0..0, state, sequence)
@ -172,69 +198,169 @@ defmodule Indexer.BlockFetcherTest do
assert address.fetched_balance_block_number == 0
end
test "can import range with all synchronous imported schemas", %{state: state} do
test "can import range with all synchronous imported schemas", %{state: state, variant: variant} do
{:ok, sequence} = Sequence.start_link([], 0, 1)
assert {:ok,
%{
addresses: [
%Explorer.Chain.Hash{
byte_count: 20,
bytes:
<<139, 243, 141, 71, 100, 146, 144, 100, 242, 212, 211, 165, 101, 32, 167, 106, 179, 223, 65, 91>>
} = first_address_hash,
%Explorer.Chain.Hash{
byte_count: 20,
bytes:
<<232, 221, 197, 199, 162, 210, 240, 215, 169, 121, 132, 89, 192, 16, 79, 223, 94, 152, 122, 202>>
} = second_address_hash
],
blocks: [
%Explorer.Chain.Hash{
byte_count: 32,
bytes:
<<246, 180, 184, 200, 141, 243, 235, 210, 82, 236, 71, 99, 40, 51, 77, 192, 38, 207, 102, 96, 106,
132, 251, 118, 155, 61, 60, 188, 204, 132, 113, 189>>
}
],
logs: [
%{
index: 0,
transaction_hash: %Explorer.Chain.Hash{
byte_count: 32,
bytes:
<<83, 189, 136, 72, 114, 222, 62, 72, 134, 146, 136, 27, 174, 236, 38, 46, 123, 149, 35, 77, 57,
101, 36, 140, 57, 254, 153, 47, 255, 212, 51, 229>>
}
}
],
transactions: [
%Explorer.Chain.Hash{
byte_count: 32,
bytes:
<<83, 189, 136, 72, 114, 222, 62, 72, 134, 146, 136, 27, 174, 236, 38, 46, 123, 149, 35, 77, 57,
101, 36, 140, 57, 254, 153, 47, 255, 212, 51, 229>>
}
]
}} = BlockFetcher.import_range(@first_full_block_number..@first_full_block_number, state, sequence)
wait_for_tasks(InternalTransactionFetcher)
wait_for_tasks(AddressBalanceFetcher)
assert Repo.aggregate(Block, :count, :hash) == 1
assert Repo.aggregate(Address, :count, :hash) == 2
assert Repo.aggregate(Log, :count, :id) == 1
assert Repo.aggregate(Transaction, :count, :hash) == 1
first_address = Repo.get!(Address, first_address_hash)
case variant do
EthereumJSONRPC.Geth ->
block_number = 48230
assert first_address.fetched_balance == %Wei{value: Decimal.new(1)}
assert first_address.fetched_balance_block_number == @first_full_block_number
second_address = Repo.get!(Address, second_address_hash)
assert second_address.fetched_balance == %Wei{value: Decimal.new(252_460_837_000_000_000_000_000_000)}
assert second_address.fetched_balance_block_number == @first_full_block_number
assert {:ok,
%{
addresses: [
%Explorer.Chain.Hash{
byte_count: 20,
bytes:
<<55, 52, 203, 24, 116, 145, 237, 231, 19, 174, 91, 59, 45, 18, 40, 74, 244, 107, 129, 1>>
} = first_address_hash,
%Explorer.Chain.Hash{
byte_count: 20,
bytes:
<<89, 47, 120, 202, 98, 102, 132, 20, 109, 56, 18, 133, 202, 0, 221, 145, 179, 117, 253, 17>>
} = second_address_hash,
%Explorer.Chain.Hash{
byte_count: 20,
bytes:
<<187, 123, 130, 135, 243, 240, 169, 51, 71, 74, 121, 234, 228, 44, 188, 169, 119, 121, 17,
113>>
} = third_address_hash,
%Explorer.Chain.Hash{
byte_count: 20,
bytes:
<<210, 193, 91, 230, 52, 135, 86, 246, 145, 187, 152, 246, 13, 254, 190, 97, 230, 190, 59,
86>>
} = fourth_address_hash,
%Explorer.Chain.Hash{
byte_count: 20,
bytes:
<<221, 47, 30, 110, 73, 130, 2, 232, 109, 143, 84, 66, 175, 89, 101, 128, 164, 240, 60, 44>>
} = fifth_address_hash
],
blocks: [
%Explorer.Chain.Hash{
byte_count: 32,
bytes:
<<209, 52, 30, 145, 228, 166, 153, 192, 47, 187, 24, 4, 84, 20, 80, 18, 144, 134, 68, 198,
200, 119, 77, 16, 251, 182, 96, 253, 27, 146, 104, 176>>
}
],
logs: [],
transactions: [
%Explorer.Chain.Hash{
byte_count: 32,
bytes:
<<76, 188, 236, 37, 153, 153, 224, 115, 252, 79, 176, 224, 228, 166, 18, 66, 94, 61, 115, 57,
47, 162, 37, 255, 36, 96, 161, 238, 171, 66, 99, 10>>
},
%Explorer.Chain.Hash{
byte_count: 32,
bytes:
<<240, 237, 34, 44, 16, 174, 248, 135, 4, 196, 15, 198, 34, 220, 218, 174, 13, 208, 242, 122,
154, 143, 4, 28, 171, 95, 190, 255, 254, 174, 75, 182>>
}
]
}} = BlockFetcher.import_range(block_number..block_number, state, sequence)
wait_for_tasks(InternalTransactionFetcher)
wait_for_tasks(AddressBalanceFetcher)
assert Repo.aggregate(Block, :count, :hash) == 1
assert Repo.aggregate(Address, :count, :hash) == 5
assert Repo.aggregate(Log, :count, :id) == 0
assert Repo.aggregate(Transaction, :count, :hash) == 2
first_address = Repo.get!(Address, first_address_hash)
assert first_address.fetched_balance == %Wei{value: Decimal.new(1_999_953_415_287_753_599_000)}
assert first_address.fetched_balance_block_number == block_number
second_address = Repo.get!(Address, second_address_hash)
assert second_address.fetched_balance == %Wei{value: Decimal.new(50_000_000_000_000_000)}
assert second_address.fetched_balance_block_number == block_number
third_address = Repo.get!(Address, third_address_hash)
assert third_address.fetched_balance == %Wei{value: Decimal.new(30_827_986_037_499_360_709_544)}
assert third_address.fetched_balance_block_number == block_number
fourth_address = Repo.get!(Address, fourth_address_hash)
assert fourth_address.fetched_balance == %Wei{value: Decimal.new(500_000_000_001_437_727_304)}
assert fourth_address.fetched_balance_block_number == block_number
fifth_address = Repo.get!(Address, fifth_address_hash)
assert fifth_address.fetched_balance == %Wei{value: Decimal.new(930_417_572_224_879_702_000)}
assert fifth_address.fetched_balance_block_number == block_number
EthereumJSONRPC.Parity ->
assert {:ok,
%{
addresses: [
%Explorer.Chain.Hash{
byte_count: 20,
bytes:
<<139, 243, 141, 71, 100, 146, 144, 100, 242, 212, 211, 165, 101, 32, 167, 106, 179, 223, 65,
91>>
} = first_address_hash,
%Explorer.Chain.Hash{
byte_count: 20,
bytes:
<<232, 221, 197, 199, 162, 210, 240, 215, 169, 121, 132, 89, 192, 16, 79, 223, 94, 152, 122,
202>>
} = second_address_hash
],
blocks: [
%Explorer.Chain.Hash{
byte_count: 32,
bytes:
<<246, 180, 184, 200, 141, 243, 235, 210, 82, 236, 71, 99, 40, 51, 77, 192, 38, 207, 102, 96,
106, 132, 251, 118, 155, 61, 60, 188, 204, 132, 113, 189>>
}
],
logs: [
%{
index: 0,
transaction_hash: %Explorer.Chain.Hash{
byte_count: 32,
bytes:
<<83, 189, 136, 72, 114, 222, 62, 72, 134, 146, 136, 27, 174, 236, 38, 46, 123, 149, 35, 77,
57, 101, 36, 140, 57, 254, 153, 47, 255, 212, 51, 229>>
}
}
],
transactions: [
%Explorer.Chain.Hash{
byte_count: 32,
bytes:
<<83, 189, 136, 72, 114, 222, 62, 72, 134, 146, 136, 27, 174, 236, 38, 46, 123, 149, 35, 77,
57, 101, 36, 140, 57, 254, 153, 47, 255, 212, 51, 229>>
}
]
}} = BlockFetcher.import_range(@first_full_block_number..@first_full_block_number, state, sequence)
wait_for_tasks(InternalTransactionFetcher)
wait_for_tasks(AddressBalanceFetcher)
assert Repo.aggregate(Block, :count, :hash) == 1
assert Repo.aggregate(Address, :count, :hash) == 2
assert Repo.aggregate(Log, :count, :id) == 1
assert Repo.aggregate(Transaction, :count, :hash) == 1
first_address = Repo.get!(Address, first_address_hash)
assert first_address.fetched_balance == %Wei{value: Decimal.new(1)}
assert first_address.fetched_balance_block_number == @first_full_block_number
second_address = Repo.get!(Address, second_address_hash)
assert second_address.fetched_balance == %Wei{value: Decimal.new(252_460_837_000_000_000_000_000_000)}
assert second_address.fetched_balance_block_number == @first_full_block_number
_ ->
raise ArgumentError, "Unsupport variant (#{variant})"
end
end
end
@ -292,7 +418,7 @@ defmodule Indexer.BlockFetcherTest do
end
defp wait_for_tasks(buffered_task) do
wait_until(5000, fn ->
wait_until(10_000, fn ->
counts = BufferedTask.debug_count(buffered_task)
counts.buffer == 0 and counts.tasks == 0
end)

@ -3,27 +3,28 @@ defmodule Indexer.InternalTransactionFetcherTest do
import ExUnit.CaptureLog
alias Explorer.Chain.Transaction
alias Indexer.{AddressBalanceFetcherCase, InternalTransactionFetcher, PendingTransactionFetcher}
alias Indexer.{AddressBalanceFetcherCase, InternalTransactionFetcher}
@moduletag :capture_log
test "does not try to fetch pending transactions from Indexer.PendingTransactionFetcher" do
start_supervised!({Task.Supervisor, name: Indexer.TaskSupervisor})
AddressBalanceFetcherCase.start_supervised!()
start_supervised!(PendingTransactionFetcher)
if EthereumJSONRPC.config(:variant) != EthereumJSONRPC.Geth do
test "does not try to fetch pending transactions from Indexer.PendingTransactionFetcher" do
start_supervised!({Task.Supervisor, name: Indexer.TaskSupervisor})
AddressBalanceFetcherCase.start_supervised!()
start_supervised!(Indexer.PendingTransactionFetcher)
wait_for_results(fn ->
Repo.one!(from(transaction in Transaction, where: is_nil(transaction.block_hash), limit: 1))
end)
wait_for_results(fn ->
Repo.one!(from(transaction in Explorer.Chain.Transaction, where: is_nil(transaction.block_hash), limit: 1))
end)
:transaction
|> insert(hash: "0x3a3eb134e6792ce9403ea4188e5e79693de9e4c94e499db132be086400da79e6")
|> with_block()
:transaction
|> insert(hash: "0x3a3eb134e6792ce9403ea4188e5e79693de9e4c94e499db132be086400da79e6")
|> with_block()
hash_strings = InternalTransactionFetcher.init([], fn hash_string, acc -> [hash_string | acc] end)
hash_strings = InternalTransactionFetcher.init([], fn hash_string, acc -> [hash_string | acc] end)
assert :ok = InternalTransactionFetcher.run(hash_strings, 0)
assert :ok = InternalTransactionFetcher.run(hash_strings, 0)
end
end
describe "init/2" do
@ -81,22 +82,24 @@ defmodule Indexer.InternalTransactionFetcherTest do
"""
end
test "duplicate transaction hashes only retry uniques" do
start_supervised!({Task.Supervisor, name: Indexer.TaskSupervisor})
AddressBalanceFetcherCase.start_supervised!()
# not a real transaction hash, so that it fails
insert(:transaction, hash: "0x0000000000000000000000000000000000000000000000000000000000000001")
assert InternalTransactionFetcher.run(
[
%{block_number: 1, hash_data: "0x0000000000000000000000000000000000000000000000000000000000000001"},
%{block_number: 1, hash_data: "0x0000000000000000000000000000000000000000000000000000000000000001"}
],
0
) ==
{:retry,
[%{block_number: 1, hash_data: "0x0000000000000000000000000000000000000000000000000000000000000001"}]}
if EthereumJSONRPC.config(:variant) != EthereumJSONRPC.Geth do
test "duplicate transaction hashes only retry uniques" do
start_supervised!({Task.Supervisor, name: Indexer.TaskSupervisor})
AddressBalanceFetcherCase.start_supervised!()
# not a real transaction hash, so that it fails
insert(:transaction, hash: "0x0000000000000000000000000000000000000000000000000000000000000001")
assert InternalTransactionFetcher.run(
[
%{block_number: 1, hash_data: "0x0000000000000000000000000000000000000000000000000000000000000001"},
%{block_number: 1, hash_data: "0x0000000000000000000000000000000000000000000000000000000000000001"}
],
0
) ==
{:retry,
[%{block_number: 1, hash_data: "0x0000000000000000000000000000000000000000000000000000000000000001"}]}
end
end
end
end

@ -2,22 +2,24 @@ defmodule Indexer.PendingTransactionFetcherTest do
# `async: false` due to use of named GenServer
use Explorer.DataCase, async: false
alias Explorer.Chain.Transaction
alias Indexer.PendingTransactionFetcher
if EthereumJSONRPC.config(:variant) != EthereumJSONRPC.Geth do
describe "start_link/1" do
# this test may fail if Sokol so low volume that the pending transactions are empty for too long
test "starts fetching pending transactions" do
alias Explorer.Chain.Transaction
alias Indexer.PendingTransactionFetcher
describe "start_link/1" do
# this test may fail if Sokol so low volume that the pending transactions are empty for too long
test "starts fetching pending transactions" do
assert Repo.aggregate(Transaction, :count, :hash) == 0
assert Repo.aggregate(Transaction, :count, :hash) == 0
start_supervised!({Task.Supervisor, name: Indexer.TaskSupervisor})
start_supervised!(PendingTransactionFetcher)
start_supervised!({Task.Supervisor, name: Indexer.TaskSupervisor})
start_supervised!(PendingTransactionFetcher)
wait_for_results(fn ->
Repo.one!(from(transaction in Transaction, where: is_nil(transaction.block_hash), limit: 1))
end)
wait_for_results(fn ->
Repo.one!(from(transaction in Transaction, where: is_nil(transaction.block_hash), limit: 1))
end)
assert Repo.aggregate(Transaction, :count, :hash) >= 1
assert Repo.aggregate(Transaction, :count, :hash) >= 1
end
end
end
end

Loading…
Cancel
Save