Catalog token transfers and tokens during indexing (#484)
* Fetch token information asynchronously during indexing * Include token transfers imports in main importer * Extract token transfers when importing blocks while indexing * Move ecto to explorer deps * Update References to the Indexer namespace * Add token insertions with main importer * Import token address/type during block indexingpull/495/head
Alex Garibay
6 years ago
committed by
GitHub
parent
9188ffa14b
commit
8b16f81b73
@ -1,4 +1,4 @@ |
|||||||
<%= link(@transaction_hash, |
<%= link(@transaction_hash, |
||||||
to: transaction_path(ExplorerWeb.Endpoint, :show, @locale, @transaction_hash), |
to: transaction_path(ExplorerWeb.Endpoint, :show, @locale, @transaction_hash), |
||||||
"data-test": "transaction_hash_link", |
"data-test": "transaction_hash_link", |
||||||
"class": "text-truncate") %> |
class: "text-truncate") %> |
||||||
|
|||||||
@ -0,0 +1,162 @@ |
|||||||
|
defmodule Indexer.TokenFetcher do |
||||||
|
@moduledoc """ |
||||||
|
Fetches information about a token. |
||||||
|
""" |
||||||
|
|
||||||
|
alias Explorer.Chain |
||||||
|
alias Explorer.Chain.Token |
||||||
|
alias Explorer.Chain.Hash.Address |
||||||
|
alias Explorer.SmartContract.Reader |
||||||
|
alias Indexer.BufferedTask |
||||||
|
|
||||||
|
@behaviour BufferedTask |
||||||
|
|
||||||
|
@defaults [ |
||||||
|
flush_interval: 300, |
||||||
|
max_batch_size: 1, |
||||||
|
max_concurrency: 10, |
||||||
|
init_chunk_size: 1, |
||||||
|
task_supervisor: Indexer.TaskSupervisor |
||||||
|
] |
||||||
|
|
||||||
|
@contract_abi [ |
||||||
|
%{ |
||||||
|
"constant" => true, |
||||||
|
"inputs" => [], |
||||||
|
"name" => "name", |
||||||
|
"outputs" => [ |
||||||
|
%{ |
||||||
|
"name" => "", |
||||||
|
"type" => "string" |
||||||
|
} |
||||||
|
], |
||||||
|
"payable" => false, |
||||||
|
"type" => "function" |
||||||
|
}, |
||||||
|
%{ |
||||||
|
"constant" => true, |
||||||
|
"inputs" => [], |
||||||
|
"name" => "decimals", |
||||||
|
"outputs" => [ |
||||||
|
%{ |
||||||
|
"name" => "", |
||||||
|
"type" => "uint8" |
||||||
|
} |
||||||
|
], |
||||||
|
"payable" => false, |
||||||
|
"type" => "function" |
||||||
|
}, |
||||||
|
%{ |
||||||
|
"constant" => true, |
||||||
|
"inputs" => [], |
||||||
|
"name" => "totalSupply", |
||||||
|
"outputs" => [ |
||||||
|
%{ |
||||||
|
"name" => "", |
||||||
|
"type" => "uint256" |
||||||
|
} |
||||||
|
], |
||||||
|
"payable" => false, |
||||||
|
"type" => "function" |
||||||
|
}, |
||||||
|
%{ |
||||||
|
"constant" => true, |
||||||
|
"inputs" => [], |
||||||
|
"name" => "symbol", |
||||||
|
"outputs" => [ |
||||||
|
%{ |
||||||
|
"name" => "", |
||||||
|
"type" => "string" |
||||||
|
} |
||||||
|
], |
||||||
|
"payable" => false, |
||||||
|
"type" => "function" |
||||||
|
} |
||||||
|
] |
||||||
|
|
||||||
|
@doc false |
||||||
|
def child_spec(provided_opts) do |
||||||
|
{state, mergable_opts} = Keyword.pop(provided_opts, :json_rpc_named_arguments) |
||||||
|
|
||||||
|
unless state do |
||||||
|
raise ArgumentError, |
||||||
|
":json_rpc_named_arguments must be provided to `#{__MODULE__}.child_spec " <> |
||||||
|
"to allow for json_rpc calls when running." |
||||||
|
end |
||||||
|
|
||||||
|
opts = |
||||||
|
@defaults |
||||||
|
|> Keyword.merge(mergable_opts) |
||||||
|
|> Keyword.put(:state, state) |
||||||
|
|
||||||
|
Supervisor.child_spec({BufferedTask, {__MODULE__, opts}}, id: __MODULE__) |
||||||
|
end |
||||||
|
|
||||||
|
@impl BufferedTask |
||||||
|
def init(initial_acc, reducer, _) do |
||||||
|
{:ok, acc} = |
||||||
|
Chain.stream_uncataloged_token_contract_address_hashes(initial_acc, fn address, acc -> |
||||||
|
reducer.(address, acc) |
||||||
|
end) |
||||||
|
|
||||||
|
acc |
||||||
|
end |
||||||
|
|
||||||
|
@impl BufferedTask |
||||||
|
def run([token_contract_address], _, json_rpc_named_arguments) do |
||||||
|
case Chain.token_from_address_hash(token_contract_address) do |
||||||
|
{:ok, %Token{cataloged: false} = token} -> |
||||||
|
catalog_token(token, json_rpc_named_arguments) |
||||||
|
|
||||||
|
{:ok, _} -> |
||||||
|
:ok |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
@doc """ |
||||||
|
Fetches token data asynchronously given a list of `t:Explorer.Chain.Token.t/0`s. |
||||||
|
""" |
||||||
|
@spec async_fetch([Address.t()]) :: :ok |
||||||
|
def async_fetch(token_contract_addresses) do |
||||||
|
BufferedTask.buffer(__MODULE__, token_contract_addresses) |
||||||
|
end |
||||||
|
|
||||||
|
defp catalog_token(%Token{contract_address_hash: contract_address_hash} = token, json_rpc_named_arguments) do |
||||||
|
contract_functions = %{ |
||||||
|
"totalSupply" => [], |
||||||
|
"decimals" => [], |
||||||
|
"name" => [], |
||||||
|
"symbol" => [] |
||||||
|
} |
||||||
|
|
||||||
|
token_contract_results = |
||||||
|
Reader.query_unverified_contract( |
||||||
|
contract_address_hash, |
||||||
|
@contract_abi, |
||||||
|
contract_functions, |
||||||
|
json_rpc_named_arguments: json_rpc_named_arguments |
||||||
|
) |
||||||
|
|
||||||
|
token_params = format_token_params(token, token_contract_results) |
||||||
|
|
||||||
|
{:ok, %{tokens: [_]}} = Chain.import(%{tokens: %{params: [token_params], on_conflict: :replace_all}}) |
||||||
|
:ok |
||||||
|
end |
||||||
|
|
||||||
|
def format_token_params(token, token_contract_data) do |
||||||
|
token_contract_data = |
||||||
|
for {function_name, {:ok, function_data}} <- token_contract_data, into: %{} do |
||||||
|
{atomized_key(function_name), function_data} |
||||||
|
end |
||||||
|
|
||||||
|
token |
||||||
|
|> Map.from_struct() |
||||||
|
|> Map.put(:cataloged, true) |
||||||
|
|> Map.merge(token_contract_data) |
||||||
|
end |
||||||
|
|
||||||
|
defp atomized_key("decimals"), do: :decimals |
||||||
|
defp atomized_key("name"), do: :name |
||||||
|
defp atomized_key("symbol"), do: :symbol |
||||||
|
defp atomized_key("totalSupply"), do: :total_supply |
||||||
|
end |
||||||
@ -0,0 +1,85 @@ |
|||||||
|
defmodule Indexer.TokenTransfers do |
||||||
|
@moduledoc """ |
||||||
|
Helper functions for transforming data for ERC-20 and ERC-721 token transfers. |
||||||
|
""" |
||||||
|
|
||||||
|
alias ABI.TypeDecoder |
||||||
|
alias Explorer.Chain.TokenTransfer |
||||||
|
|
||||||
|
@doc """ |
||||||
|
Returns a list of token transfers given a list of logs. |
||||||
|
""" |
||||||
|
def from_log_params(logs) do |
||||||
|
initial_acc = %{tokens: [], token_transfers: []} |
||||||
|
|
||||||
|
logs |
||||||
|
|> Enum.filter(&(&1.first_topic == unquote(TokenTransfer.constant()))) |
||||||
|
|> Enum.reduce(initial_acc, &do_from_log_params/2) |
||||||
|
end |
||||||
|
|
||||||
|
defp do_from_log_params(log, %{tokens: tokens, token_transfers: token_transfers}) do |
||||||
|
{token, token_transfer} = parse_params(log) |
||||||
|
|
||||||
|
%{ |
||||||
|
tokens: [token | tokens], |
||||||
|
token_transfers: [token_transfer | token_transfers] |
||||||
|
} |
||||||
|
end |
||||||
|
|
||||||
|
# ERC-20 token transfer |
||||||
|
defp parse_params(%{fourth_topic: nil} = log) do |
||||||
|
token_transfer = %{ |
||||||
|
amount: Decimal.new(convert_to_integer(log.data)), |
||||||
|
block_number: log.block_number, |
||||||
|
log_index: log.index, |
||||||
|
from_address_hash: truncate_address_hash(log.second_topic), |
||||||
|
to_address_hash: truncate_address_hash(log.third_topic), |
||||||
|
token_contract_address_hash: log.address_hash, |
||||||
|
transaction_hash: log.transaction_hash |
||||||
|
} |
||||||
|
|
||||||
|
token = %{ |
||||||
|
contract_address_hash: log.address_hash, |
||||||
|
type: "ERC-20" |
||||||
|
} |
||||||
|
|
||||||
|
{token, token_transfer} |
||||||
|
end |
||||||
|
|
||||||
|
# ERC-721 token transfer |
||||||
|
defp parse_params(%{fourth_topic: fourth_topic} = log) when not is_nil(fourth_topic) do |
||||||
|
token_transfer = %{ |
||||||
|
block_number: log.block_number, |
||||||
|
log_index: log.index, |
||||||
|
from_address_hash: truncate_address_hash(log.second_topic), |
||||||
|
to_address_hash: truncate_address_hash(log.third_topic), |
||||||
|
token_contract_address_hash: log.address_hash, |
||||||
|
token_id: convert_to_integer(fourth_topic), |
||||||
|
transaction_hash: log.transaction_hash |
||||||
|
} |
||||||
|
|
||||||
|
token = %{ |
||||||
|
contract_address_hash: log.address_hash, |
||||||
|
type: "ERC-721" |
||||||
|
} |
||||||
|
|
||||||
|
{token, token_transfer} |
||||||
|
end |
||||||
|
|
||||||
|
defp truncate_address_hash(nil), do: "0x0000000000000000000000000000000000000000" |
||||||
|
|
||||||
|
defp truncate_address_hash("0x000000000000000000000000" <> truncated_hash) do |
||||||
|
"0x#{truncated_hash}" |
||||||
|
end |
||||||
|
|
||||||
|
defp convert_to_integer("0x"), do: 0 |
||||||
|
|
||||||
|
defp convert_to_integer("0x" <> encoded_integer) do |
||||||
|
[value] = |
||||||
|
encoded_integer |
||||||
|
|> Base.decode16!(case: :mixed) |
||||||
|
|> TypeDecoder.decode_raw([{:uint, 256}]) |
||||||
|
|
||||||
|
value |
||||||
|
end |
||||||
|
end |
||||||
@ -0,0 +1,78 @@ |
|||||||
|
defmodule Indexer.TokenFetcherTest do |
||||||
|
use EthereumJSONRPC.Case |
||||||
|
use Explorer.DataCase |
||||||
|
|
||||||
|
import Mox |
||||||
|
|
||||||
|
alias Explorer.Chain |
||||||
|
alias Explorer.Chain.Token |
||||||
|
alias Indexer.TokenFetcher |
||||||
|
|
||||||
|
setup :verify_on_exit! |
||||||
|
|
||||||
|
describe "init/3" do |
||||||
|
test "returns uncataloged tokens", %{json_rpc_named_arguments: json_rpc_named_arguments} do |
||||||
|
insert(:token, cataloged: true) |
||||||
|
%Token{contract_address_hash: uncatalog_address} = insert(:token, cataloged: false) |
||||||
|
|
||||||
|
assert TokenFetcher.init([], &[&1 | &2], json_rpc_named_arguments) == [uncatalog_address] |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
describe "run/3" do |
||||||
|
test "skips tokens that have already been cataloged", %{json_rpc_named_arguments: json_rpc_named_arguments} do |
||||||
|
expect(EthereumJSONRPC.Mox, :json_rpc, 0, fn _, _ -> :ok end) |
||||||
|
%Token{contract_address_hash: contract_address_hash} = insert(:token, cataloged: true) |
||||||
|
assert TokenFetcher.run([contract_address_hash], 0, json_rpc_named_arguments) == :ok |
||||||
|
end |
||||||
|
|
||||||
|
test "catalogs tokens that haven't been cataloged", %{json_rpc_named_arguments: json_rpc_named_arguments} do |
||||||
|
token = insert(:token, name: nil, symbol: nil, total_supply: nil, decimals: nil, cataloged: false) |
||||||
|
contract_address_hash = token.contract_address_hash |
||||||
|
|
||||||
|
if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do |
||||||
|
expect( |
||||||
|
EthereumJSONRPC.Mox, |
||||||
|
:json_rpc, |
||||||
|
1, |
||||||
|
fn [%{id: "decimals"}, %{id: "name"}, %{id: "symbol"}, %{id: "totalSupply"}], _opts -> |
||||||
|
{:ok, |
||||||
|
[ |
||||||
|
%{ |
||||||
|
id: "decimals", |
||||||
|
result: "0x0000000000000000000000000000000000000000000000000000000000000012" |
||||||
|
}, |
||||||
|
%{ |
||||||
|
id: "name", |
||||||
|
result: |
||||||
|
"0x0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000642616e636f720000000000000000000000000000000000000000000000000000" |
||||||
|
}, |
||||||
|
%{ |
||||||
|
id: "symbol", |
||||||
|
result: |
||||||
|
"0x00000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000003424e540000000000000000000000000000000000000000000000000000000000" |
||||||
|
}, |
||||||
|
%{ |
||||||
|
id: "totalSupply", |
||||||
|
result: "0x0000000000000000000000000000000000000000000000000de0b6b3a7640000" |
||||||
|
} |
||||||
|
]} |
||||||
|
end |
||||||
|
) |
||||||
|
|
||||||
|
assert TokenFetcher.run([contract_address_hash], 0, json_rpc_named_arguments) == :ok |
||||||
|
|
||||||
|
expected_supply = Decimal.new(1_000_000_000_000_000_000) |
||||||
|
|
||||||
|
assert {:ok, |
||||||
|
%Token{ |
||||||
|
name: "Bancor", |
||||||
|
symbol: "BNT", |
||||||
|
total_supply: ^expected_supply, |
||||||
|
decimals: 18, |
||||||
|
cataloged: true |
||||||
|
}} = Chain.token_from_address_hash(contract_address_hash) |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
@ -0,0 +1,88 @@ |
|||||||
|
defmodule Indexer.TokenTransfersTest do |
||||||
|
use ExUnit.Case |
||||||
|
|
||||||
|
alias Indexer.TokenTransfers |
||||||
|
|
||||||
|
describe "from_log_params/2" do |
||||||
|
test "from_log_params/2 parses logs for tokens and token transfers" do |
||||||
|
[log_1, _log_2, log_3] = |
||||||
|
logs = [ |
||||||
|
%{ |
||||||
|
address_hash: "0xf2eec76e45b328df99a34fa696320a262cb92154", |
||||||
|
block_number: 3_530_917, |
||||||
|
data: "0x000000000000000000000000000000000000000000000000ebec21ee1da40000", |
||||||
|
first_topic: "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", |
||||||
|
fourth_topic: nil, |
||||||
|
index: 8, |
||||||
|
second_topic: "0x000000000000000000000000556813d9cc20acfe8388af029a679d34a63388db", |
||||||
|
third_topic: "0x00000000000000000000000092148dd870fa1b7c4700f2bd7f44238821c26f73", |
||||||
|
transaction_hash: "0x43dfd761974e8c3351d285ab65bee311454eb45b149a015fe7804a33252f19e5", |
||||||
|
type: "mined" |
||||||
|
}, |
||||||
|
%{ |
||||||
|
address_hash: "0x6ea5ec9cb832e60b6b1654f5826e9be638f276a5", |
||||||
|
block_number: 3_586_935, |
||||||
|
data: "0x", |
||||||
|
first_topic: "0x55e10366a5f552746106978b694d7ef3bbddec06bd5f9b9d15ad46f475c653ef", |
||||||
|
fourth_topic: "0xc89efdaa54c0f20c7adf612882df0950f5a951637e0307cdcb4c672f298b8bc6", |
||||||
|
index: 0, |
||||||
|
second_topic: "0x00000000000000000000000063b0595bb7a0b7edd0549c9557a0c8aee6da667b", |
||||||
|
third_topic: "0x000000000000000000000000f3089e15d0c23c181d7f98b0878b560bfe193a1d", |
||||||
|
transaction_hash: "0x8425a9b81a9bd1c64861110c1a453b84719cb0361d6fa0db68abf7611b9a890e", |
||||||
|
type: "mined" |
||||||
|
}, |
||||||
|
%{ |
||||||
|
address_hash: "0x91932e8c6776fb2b04abb71874a7988747728bb2", |
||||||
|
block_number: 3_664_064, |
||||||
|
data: "0x000000000000000000000000000000000000000000000000ebec21ee1da40000", |
||||||
|
first_topic: "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", |
||||||
|
fourth_topic: "0x00000000000000000000000000000000000000000000000000000000000000b7", |
||||||
|
index: 1, |
||||||
|
second_topic: "0x0000000000000000000000009851ba177554eb07271ac230a137551e6dd0aa84", |
||||||
|
third_topic: "0x000000000000000000000000dccb72afee70e60b0c1226288fe86c01b953e8ac", |
||||||
|
transaction_hash: "0x4011d9a930a3da620321589a54dc0ca3b88216b4886c7a7c3aaad1fb17702d35", |
||||||
|
type: "mined" |
||||||
|
} |
||||||
|
] |
||||||
|
|
||||||
|
expected = %{ |
||||||
|
tokens: [ |
||||||
|
%{ |
||||||
|
contract_address_hash: log_3.address_hash, |
||||||
|
type: "ERC-721" |
||||||
|
}, |
||||||
|
%{ |
||||||
|
contract_address_hash: log_1.address_hash, |
||||||
|
type: "ERC-20" |
||||||
|
} |
||||||
|
], |
||||||
|
token_transfers: [ |
||||||
|
%{ |
||||||
|
block_number: log_3.block_number, |
||||||
|
log_index: log_3.index, |
||||||
|
from_address_hash: truncated_hash(log_3.second_topic), |
||||||
|
to_address_hash: truncated_hash(log_3.third_topic), |
||||||
|
token_contract_address_hash: log_3.address_hash, |
||||||
|
token_id: 183, |
||||||
|
transaction_hash: log_3.transaction_hash |
||||||
|
}, |
||||||
|
%{ |
||||||
|
amount: Decimal.new(17_000_000_000_000_000_000), |
||||||
|
block_number: log_1.block_number, |
||||||
|
log_index: log_1.index, |
||||||
|
from_address_hash: truncated_hash(log_1.second_topic), |
||||||
|
to_address_hash: truncated_hash(log_1.third_topic), |
||||||
|
token_contract_address_hash: log_1.address_hash, |
||||||
|
transaction_hash: log_1.transaction_hash |
||||||
|
} |
||||||
|
] |
||||||
|
} |
||||||
|
|
||||||
|
assert TokenTransfers.from_log_params(logs) == expected |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
defp truncated_hash("0x000000000000000000000000" <> rest) do |
||||||
|
"0x" <> rest |
||||||
|
end |
||||||
|
end |
||||||
@ -0,0 +1,16 @@ |
|||||||
|
defmodule Indexer.TokenFetcherCase do |
||||||
|
alias Indexer.TokenFetcher |
||||||
|
|
||||||
|
def start_supervised!(options \\ []) when is_list(options) do |
||||||
|
options |
||||||
|
|> Keyword.merge( |
||||||
|
flush_interval: 50, |
||||||
|
init_chunk_size: 1, |
||||||
|
max_batch_size: 1, |
||||||
|
max_concurrency: 1, |
||||||
|
name: TokenFetcher |
||||||
|
) |
||||||
|
|> TokenFetcher.child_spec() |
||||||
|
|> ExUnit.Callbacks.start_supervised!() |
||||||
|
end |
||||||
|
end |
||||||
Loading…
Reference in new issue