From 06ce9ab5c8af44bb80375c870fd7ea81d7bc0d26 Mon Sep 17 00:00:00 2001 From: William Sanches Date: Tue, 13 Nov 2018 17:03:10 -0200 Subject: [PATCH 1/7] Add chain function to get cataloged tokens --- apps/explorer/lib/explorer/chain.ex | 26 ++++++++++++++++++++++ apps/explorer/test/explorer/chain_test.exs | 6 +++++ 2 files changed, 32 insertions(+) diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index a13378674b..45214149ce 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -1939,6 +1939,32 @@ defmodule Explorer.Chain do ) end + @doc """ + Streams a list of token contract addresses that have been cataloged. + """ + @spec stream_cataloged_token_contract_address_hashes( + initial :: accumulator, + reducer :: (entry :: Hash.Address.t(), accumulator -> accumulator) + ) :: {:ok, accumulator} + when accumulator: term() + def stream_cataloged_token_contract_address_hashes(initial_acc, reducer) when is_function(reducer, 2) do + Repo.transaction( + fn -> + query = + from( + token in Token, + where: token.cataloged == true, + select: token.contract_address_hash + ) + + query + |> Repo.stream(timeout: :infinity) + |> Enum.reduce(initial_acc, reducer) + end, + timeout: :infinity + ) + end + @doc """ Returns a list of block numbers token transfer `t:Log.t/0`s that don't have an associated `t:TokenTransfer.t/0` record. diff --git a/apps/explorer/test/explorer/chain_test.exs b/apps/explorer/test/explorer/chain_test.exs index 2d1d56e44d..6d8f2f633c 100644 --- a/apps/explorer/test/explorer/chain_test.exs +++ b/apps/explorer/test/explorer/chain_test.exs @@ -2756,6 +2756,12 @@ defmodule Explorer.ChainTest do assert Chain.stream_uncataloged_token_contract_address_hashes([], &[&1 | &2]) == {:ok, [uncatalog_address]} end + test "stream_cataloged_token_contract_address_hashes/2 reduces with given reducer and accumulator" do + %Token{contract_address_hash: catalog_address} = insert(:token, cataloged: true) + insert(:token, cataloged: false) + assert Chain.stream_cataloged_token_contract_address_hashes([], &[&1 | &2]) == {:ok, [catalog_address]} + end + describe "transaction_has_token_transfers?/1" do test "returns true if transaction has token transfers" do transaction = insert(:transaction) From 3528e4dc5101e5cb32f775da23d8dbd1eff6e8f8 Mon Sep 17 00:00:00 2001 From: William Sanches Date: Tue, 13 Nov 2018 17:03:26 -0200 Subject: [PATCH 2/7] Add metadata updater --- apps/explorer/lib/explorer/chain.ex | 3 +- apps/indexer/config/config.exs | 1 + .../lib/indexer/token/metadata_updater.ex | 49 +++++++++++++++ apps/indexer/lib/indexer/token/supervisor.ex | 5 +- .../indexer/token/metadata_updater_test.exs | 61 +++++++++++++++++++ 5 files changed, 116 insertions(+), 3 deletions(-) create mode 100644 apps/indexer/lib/indexer/token/metadata_updater.ex create mode 100644 apps/indexer/test/indexer/token/metadata_updater_test.exs diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index 45214149ce..fa85587925 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -1953,8 +1953,9 @@ defmodule Explorer.Chain do query = from( token in Token, + select: token.contract_address_hash, where: token.cataloged == true, - select: token.contract_address_hash + order_by: [asc: token.updated_at] ) query diff --git a/apps/indexer/config/config.exs b/apps/indexer/config/config.exs index 43ff7c3d18..e19bcd3610 100644 --- a/apps/indexer/config/config.exs +++ b/apps/indexer/config/config.exs @@ -7,6 +7,7 @@ import Bitwise config :indexer, block_transformer: Indexer.Block.Transform.Base, ecto_repos: [Explorer.Repo], + metadata_updater_days_interval: 7, # bytes memory_limit: 1 <<< 30 diff --git a/apps/indexer/lib/indexer/token/metadata_updater.ex b/apps/indexer/lib/indexer/token/metadata_updater.ex new file mode 100644 index 0000000000..be502c64bf --- /dev/null +++ b/apps/indexer/lib/indexer/token/metadata_updater.ex @@ -0,0 +1,49 @@ +defmodule Indexer.Token.MetadataUpdater do + @moduledoc """ + Updates metadata for cataloged tokens + """ + + use GenServer + + alias Explorer.Chain + alias Explorer.Chain.Token + alias Explorer.Token.MetadataRetriever + + def start_link(_) do + GenServer.start_link(__MODULE__, :ok, name: __MODULE__) + end + + @impl true + def init(state) do + send(self(), :update_tokens) + + {:ok, state} + end + + @impl true + def handle_info(:update_tokens, state) do + {:ok, tokens} = Chain.stream_cataloged_token_contract_address_hashes([], &[&1 | &2]) + update_metadata(tokens) + + interval = Application.get_env(:indexer, :metadata_updater_days_interval) + Process.send_after(self(), :update_tokens, :timer.hours(interval) * 24) + + {:noreply, state} + end + + @doc false + def update_metadata(token_addresses) when is_list(token_addresses) do + Enum.each(token_addresses, fn address -> + case Chain.token_from_address_hash(address) do + {:ok, %Token{cataloged: true} = token} -> + update_metadata(token) + end + end) + end + + def update_metadata(%Token{contract_address_hash: contract_address_hash} = token) do + contract_functions = MetadataRetriever.get_functions_of(contract_address_hash) + + Chain.update_token(%{token | updated_at: DateTime.utc_now()}, contract_functions) + end +end diff --git a/apps/indexer/lib/indexer/token/supervisor.ex b/apps/indexer/lib/indexer/token/supervisor.ex index fd54e26f46..6c51feffa2 100644 --- a/apps/indexer/lib/indexer/token/supervisor.ex +++ b/apps/indexer/lib/indexer/token/supervisor.ex @@ -5,7 +5,7 @@ defmodule Indexer.Token.Supervisor do use Supervisor - alias Indexer.Token.Fetcher + alias Indexer.Token.{Fetcher, MetadataUpdater} def child_spec([init_arguments]) do child_spec([init_arguments, []]) @@ -30,7 +30,8 @@ defmodule Indexer.Token.Supervisor do Supervisor.init( [ {Task.Supervisor, name: Indexer.Token.TaskSupervisor}, - {Fetcher, [fetcher_arguments, [name: Fetcher]]} + {Fetcher, [fetcher_arguments, [name: Fetcher]]}, + {MetadataUpdater, [[], [name: MetadataUpdater]]} ], strategy: :one_for_one ) diff --git a/apps/indexer/test/indexer/token/metadata_updater_test.exs b/apps/indexer/test/indexer/token/metadata_updater_test.exs new file mode 100644 index 0000000000..f1caa67e20 --- /dev/null +++ b/apps/indexer/test/indexer/token/metadata_updater_test.exs @@ -0,0 +1,61 @@ +defmodule Indexer.Token.MetadataUpdaterTest do + use Explorer.DataCase + + import Mox + + alias Explorer.Chain + alias Explorer.Chain.Token + alias Indexer.Token.MetadataUpdater + + setup :verify_on_exit! + + describe "update_metadata/1" do + test "updates the metadata for a list of tokens" do + token = insert(:token, name: nil, symbol: nil, decimals: 10) + + expect( + EthereumJSONRPC.Mox, + :json_rpc, + 1, + fn [%{id: "decimals"}, %{id: "name"}, %{id: "symbol"}, %{id: "totalSupply"}], _opts -> + {:ok, + [ + %{ + id: "decimals", + result: "0x0000000000000000000000000000000000000000000000000000000000000012" + }, + %{ + id: "name", + result: + "0x0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000642616e636f720000000000000000000000000000000000000000000000000000" + }, + %{ + id: "symbol", + result: + "0x00000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000003424e540000000000000000000000000000000000000000000000000000000000" + }, + %{ + id: "totalSupply", + result: "0x0000000000000000000000000000000000000000000000000de0b6b3a7640000" + } + ]} + end + ) + + MetadataUpdater.update_metadata([token.contract_address_hash]) + + expected_supply = Decimal.new(1_000_000_000_000_000_000) + + decimals_expected = Decimal.new(18) + + assert {:ok, + %Token{ + name: "Bancor", + symbol: "BNT", + total_supply: ^expected_supply, + decimals: ^decimals_expected, + cataloged: true + }} = Chain.token_from_address_hash(token.contract_address_hash) + end + end +end From 5b14292aa9ea3ce67cce86ebe6490f7abbf26a5e Mon Sep 17 00:00:00 2001 From: William Sanches Date: Fri, 16 Nov 2018 11:10:00 -0200 Subject: [PATCH 3/7] Fix reducer function on metadata updater It was concatenating the result in the reverse order. --- apps/indexer/lib/indexer/token/metadata_updater.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/indexer/lib/indexer/token/metadata_updater.ex b/apps/indexer/lib/indexer/token/metadata_updater.ex index be502c64bf..0606b0a0b5 100644 --- a/apps/indexer/lib/indexer/token/metadata_updater.ex +++ b/apps/indexer/lib/indexer/token/metadata_updater.ex @@ -22,7 +22,7 @@ defmodule Indexer.Token.MetadataUpdater do @impl true def handle_info(:update_tokens, state) do - {:ok, tokens} = Chain.stream_cataloged_token_contract_address_hashes([], &[&1 | &2]) + {:ok, tokens} = Chain.stream_cataloged_token_contract_address_hashes([], &(&2 ++ [&1])) update_metadata(tokens) interval = Application.get_env(:indexer, :metadata_updater_days_interval) From 8ee4e62087b315e9ae2d6cedb9d01329f5a439b6 Mon Sep 17 00:00:00 2001 From: William Sanches Date: Fri, 16 Nov 2018 11:13:09 -0200 Subject: [PATCH 4/7] Refactor cataloged token query to Chain.Token --- apps/explorer/lib/explorer/chain.ex | 11 ++------ apps/explorer/lib/explorer/chain/token.ex | 13 ++++++++++ .../test/explorer/chain/token_test.exs | 16 ++++++++++++ apps/explorer/test/explorer/chain_test.exs | 25 ++++++++++++++++--- 4 files changed, 52 insertions(+), 13 deletions(-) create mode 100644 apps/explorer/test/explorer/chain/token_test.exs diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index fa85587925..d79126f9dd 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -1950,15 +1950,8 @@ defmodule Explorer.Chain do def stream_cataloged_token_contract_address_hashes(initial_acc, reducer) when is_function(reducer, 2) do Repo.transaction( fn -> - query = - from( - token in Token, - select: token.contract_address_hash, - where: token.cataloged == true, - order_by: [asc: token.updated_at] - ) - - query + Chain.Token.cataloged_tokens() + |> Ecto.Query.order_by([asc: :updated_at]) |> Repo.stream(timeout: :infinity) |> Enum.reduce(initial_acc, reducer) end, diff --git a/apps/explorer/lib/explorer/chain/token.ex b/apps/explorer/lib/explorer/chain/token.ex index 81a4c96ba9..306ded7856 100644 --- a/apps/explorer/lib/explorer/chain/token.ex +++ b/apps/explorer/lib/explorer/chain/token.ex @@ -94,4 +94,17 @@ defmodule Explorer.Chain.Token do on: tt.token_contract_address_hash == t.contract_address_hash ) end + + @doc """ + Builds an `Ecto.Query` to fetch the cataloged tokens. + + These are tokens with cataloged field set to true. + """ + def cataloged_tokens() do + from( + token in __MODULE__, + select: token.contract_address_hash, + where: token.cataloged == true + ) + end end diff --git a/apps/explorer/test/explorer/chain/token_test.exs b/apps/explorer/test/explorer/chain/token_test.exs new file mode 100644 index 0000000000..57cf42d13a --- /dev/null +++ b/apps/explorer/test/explorer/chain/token_test.exs @@ -0,0 +1,16 @@ +defmodule Explorer.Chain.TokenTest do + use Explorer.DataCase + + import Explorer.Factory + + alias Explorer.Chain + + describe "cataloged_tokens/0" do + test "filters uncataloged tokens out" do + token = insert(:token, cataloged: true) + insert(:token, cataloged: false) + + assert Repo.all(Chain.Token.cataloged_tokens()) == [token.contract_address_hash] + end + end +end diff --git a/apps/explorer/test/explorer/chain_test.exs b/apps/explorer/test/explorer/chain_test.exs index 6d8f2f633c..2feb6aeddc 100644 --- a/apps/explorer/test/explorer/chain_test.exs +++ b/apps/explorer/test/explorer/chain_test.exs @@ -2756,10 +2756,27 @@ defmodule Explorer.ChainTest do assert Chain.stream_uncataloged_token_contract_address_hashes([], &[&1 | &2]) == {:ok, [uncatalog_address]} end - test "stream_cataloged_token_contract_address_hashes/2 reduces with given reducer and accumulator" do - %Token{contract_address_hash: catalog_address} = insert(:token, cataloged: true) - insert(:token, cataloged: false) - assert Chain.stream_cataloged_token_contract_address_hashes([], &[&1 | &2]) == {:ok, [catalog_address]} + describe "stream_cataloged_token_contract_address_hashes/2" do + test "reduces with given reducer and accumulator" do + %Token{contract_address_hash: catalog_address} = insert(:token, cataloged: true) + insert(:token, cataloged: false) + assert Chain.stream_cataloged_token_contract_address_hashes([], &[&1 | &2]) == {:ok, [catalog_address]} + end + + test "sorts the tokens by updated_at in ascending order" do + today = DateTime.utc_now() + yesterday = Timex.shift(today, days: -1) + + token1 = insert(:token, %{cataloged: true, updated_at: today}) + token2 = insert(:token, %{cataloged: true, updated_at: yesterday}) + + expected_response = + [token1, token2] + |> Enum.sort(&(&1.updated_at < &2.updated_at)) + |> Enum.map(&(&1.contract_address_hash)) + + assert Chain.stream_cataloged_token_contract_address_hashes([], &(&2 ++ [&1])) == {:ok, expected_response} + end end describe "transaction_has_token_transfers?/1" do From 2df87c2cd11373e187927bd0fa86263be9d519de Mon Sep 17 00:00:00 2001 From: William Sanches Date: Fri, 16 Nov 2018 11:35:00 -0200 Subject: [PATCH 5/7] Refactor metadata_updater to get interval on start --- apps/explorer/test/explorer/chain/token_test.exs | 2 +- apps/indexer/lib/indexer/token/metadata_updater.ex | 7 +++---- apps/indexer/lib/indexer/token/supervisor.ex | 3 ++- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/explorer/test/explorer/chain/token_test.exs b/apps/explorer/test/explorer/chain/token_test.exs index 57cf42d13a..0cab67ecff 100644 --- a/apps/explorer/test/explorer/chain/token_test.exs +++ b/apps/explorer/test/explorer/chain/token_test.exs @@ -6,7 +6,7 @@ defmodule Explorer.Chain.TokenTest do alias Explorer.Chain describe "cataloged_tokens/0" do - test "filters uncataloged tokens out" do + test "filters only cataloged tokens" do token = insert(:token, cataloged: true) insert(:token, cataloged: false) diff --git a/apps/indexer/lib/indexer/token/metadata_updater.ex b/apps/indexer/lib/indexer/token/metadata_updater.ex index 0606b0a0b5..b7135dd1e8 100644 --- a/apps/indexer/lib/indexer/token/metadata_updater.ex +++ b/apps/indexer/lib/indexer/token/metadata_updater.ex @@ -9,8 +9,8 @@ defmodule Indexer.Token.MetadataUpdater do alias Explorer.Chain.Token alias Explorer.Token.MetadataRetriever - def start_link(_) do - GenServer.start_link(__MODULE__, :ok, name: __MODULE__) + def start_link(initial_state) do + GenServer.start_link(__MODULE__, initial_state, name: __MODULE__) end @impl true @@ -25,8 +25,7 @@ defmodule Indexer.Token.MetadataUpdater do {:ok, tokens} = Chain.stream_cataloged_token_contract_address_hashes([], &(&2 ++ [&1])) update_metadata(tokens) - interval = Application.get_env(:indexer, :metadata_updater_days_interval) - Process.send_after(self(), :update_tokens, :timer.hours(interval) * 24) + Process.send_after(self(), :update_tokens, :timer.hours(state.update_interval) * 24) {:noreply, state} end diff --git a/apps/indexer/lib/indexer/token/supervisor.ex b/apps/indexer/lib/indexer/token/supervisor.ex index 6c51feffa2..d50184371b 100644 --- a/apps/indexer/lib/indexer/token/supervisor.ex +++ b/apps/indexer/lib/indexer/token/supervisor.ex @@ -27,11 +27,12 @@ defmodule Indexer.Token.Supervisor do @impl Supervisor def init(fetcher_arguments) do + metadata_updater_inverval = Application.get_env(:indexer, :metadata_updater_days_interval) Supervisor.init( [ {Task.Supervisor, name: Indexer.Token.TaskSupervisor}, {Fetcher, [fetcher_arguments, [name: Fetcher]]}, - {MetadataUpdater, [[], [name: MetadataUpdater]]} + {MetadataUpdater, %{update_interval: metadata_updater_inverval}} ], strategy: :one_for_one ) From 421262a2da057223ea74b50909572e2eb2ea5e9a Mon Sep 17 00:00:00 2001 From: William Sanches Date: Fri, 16 Nov 2018 14:24:46 -0200 Subject: [PATCH 6/7] Fix credo and format issues --- apps/explorer/lib/explorer/chain.ex | 2 +- apps/explorer/lib/explorer/chain/token.ex | 2 +- apps/explorer/test/explorer/chain_test.exs | 2 +- apps/indexer/lib/indexer/token/metadata_updater.ex | 7 +++++-- apps/indexer/lib/indexer/token/supervisor.ex | 1 + 5 files changed, 9 insertions(+), 5 deletions(-) diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index d79126f9dd..a2a55dcbd5 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -1951,7 +1951,7 @@ defmodule Explorer.Chain do Repo.transaction( fn -> Chain.Token.cataloged_tokens() - |> Ecto.Query.order_by([asc: :updated_at]) + |> order_by(asc: :updated_at) |> Repo.stream(timeout: :infinity) |> Enum.reduce(initial_acc, reducer) end, diff --git a/apps/explorer/lib/explorer/chain/token.ex b/apps/explorer/lib/explorer/chain/token.ex index 306ded7856..22ba6d2504 100644 --- a/apps/explorer/lib/explorer/chain/token.ex +++ b/apps/explorer/lib/explorer/chain/token.ex @@ -100,7 +100,7 @@ defmodule Explorer.Chain.Token do These are tokens with cataloged field set to true. """ - def cataloged_tokens() do + def cataloged_tokens do from( token in __MODULE__, select: token.contract_address_hash, diff --git a/apps/explorer/test/explorer/chain_test.exs b/apps/explorer/test/explorer/chain_test.exs index 2feb6aeddc..13c0f44906 100644 --- a/apps/explorer/test/explorer/chain_test.exs +++ b/apps/explorer/test/explorer/chain_test.exs @@ -2773,7 +2773,7 @@ defmodule Explorer.ChainTest do expected_response = [token1, token2] |> Enum.sort(&(&1.updated_at < &2.updated_at)) - |> Enum.map(&(&1.contract_address_hash)) + |> Enum.map(& &1.contract_address_hash) assert Chain.stream_cataloged_token_contract_address_hashes([], &(&2 ++ [&1])) == {:ok, expected_response} end diff --git a/apps/indexer/lib/indexer/token/metadata_updater.ex b/apps/indexer/lib/indexer/token/metadata_updater.ex index b7135dd1e8..6c3e0f1357 100644 --- a/apps/indexer/lib/indexer/token/metadata_updater.ex +++ b/apps/indexer/lib/indexer/token/metadata_updater.ex @@ -22,8 +22,11 @@ defmodule Indexer.Token.MetadataUpdater do @impl true def handle_info(:update_tokens, state) do - {:ok, tokens} = Chain.stream_cataloged_token_contract_address_hashes([], &(&2 ++ [&1])) - update_metadata(tokens) + {:ok, tokens} = Chain.stream_cataloged_token_contract_address_hashes([], &[&1 | &2]) + + tokens + |> Enum.reverse() + |> update_metadata() Process.send_after(self(), :update_tokens, :timer.hours(state.update_interval) * 24) diff --git a/apps/indexer/lib/indexer/token/supervisor.ex b/apps/indexer/lib/indexer/token/supervisor.ex index d50184371b..2e7d1b6360 100644 --- a/apps/indexer/lib/indexer/token/supervisor.ex +++ b/apps/indexer/lib/indexer/token/supervisor.ex @@ -28,6 +28,7 @@ defmodule Indexer.Token.Supervisor do @impl Supervisor def init(fetcher_arguments) do metadata_updater_inverval = Application.get_env(:indexer, :metadata_updater_days_interval) + Supervisor.init( [ {Task.Supervisor, name: Indexer.Token.TaskSupervisor}, From 3c146f92e441303d8dc01d1419625073e4d3f2f3 Mon Sep 17 00:00:00 2001 From: Amanda Date: Wed, 21 Nov 2018 13:18:03 -0200 Subject: [PATCH 7/7] Add test to MetadataUpdater's GenServer Co-Authored-By: w-sanches <1727723+w-sanches@users.noreply.github.com> --- .../indexer/token/metadata_updater_test.exs | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/apps/indexer/test/indexer/token/metadata_updater_test.exs b/apps/indexer/test/indexer/token/metadata_updater_test.exs index f1caa67e20..e72097f2b4 100644 --- a/apps/indexer/test/indexer/token/metadata_updater_test.exs +++ b/apps/indexer/test/indexer/token/metadata_updater_test.exs @@ -8,6 +8,52 @@ defmodule Indexer.Token.MetadataUpdaterTest do alias Indexer.Token.MetadataUpdater setup :verify_on_exit! + setup :set_mox_global + + test "updates tokens metadata on start" do + insert(:token, name: nil, symbol: nil, decimals: 10, cataloged: true) + + expect( + EthereumJSONRPC.Mox, + :json_rpc, + 1, + fn [%{id: "decimals"}, %{id: "name"}, %{id: "symbol"}, %{id: "totalSupply"}], _opts -> + {:ok, + [ + %{ + id: "decimals", + result: "0x0000000000000000000000000000000000000000000000000000000000000012" + }, + %{ + id: "name", + result: + "0x0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000642616e636f720000000000000000000000000000000000000000000000000000" + }, + %{ + id: "symbol", + result: + "0x00000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000003424e540000000000000000000000000000000000000000000000000000000000" + }, + %{ + id: "totalSupply", + result: "0x0000000000000000000000000000000000000000000000000de0b6b3a7640000" + } + ]} + end + ) + + pid = start_supervised!({MetadataUpdater, %{update_interval: 0}}) + + wait_for_results(fn -> + updated = Repo.one!(from(t in Token, where: t.cataloged == true and not is_nil(t.name), limit: 1)) + + assert updated.name != nil + assert updated.symbol != nil + end) + + # Terminates the process so it finishes all Ecto processes. + GenServer.stop(pid) + end describe "update_metadata/1" do test "updates the metadata for a list of tokens" do