From 978af2df39c1833de288befcc2220de8ff76e9f3 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Fri, 22 Mar 2019 12:11:49 +0300 Subject: [PATCH 01/14] add transaction count cache --- .../explorer/chain/transaction_count_cache.ex | 115 ++++++++++++++++++ .../chain/transaction_count_cache_test.exs | 58 +++++++++ 2 files changed, 173 insertions(+) create mode 100644 apps/explorer/lib/explorer/chain/transaction_count_cache.ex create mode 100644 apps/explorer/test/explorer/chain/transaction_count_cache_test.exs diff --git a/apps/explorer/lib/explorer/chain/transaction_count_cache.ex b/apps/explorer/lib/explorer/chain/transaction_count_cache.ex new file mode 100644 index 0000000000..9dc6be2e68 --- /dev/null +++ b/apps/explorer/lib/explorer/chain/transaction_count_cache.ex @@ -0,0 +1,115 @@ +defmodule Explorer.Chain.TransactionCountCache do + @moduledoc """ + Cache for estimated transaction count. + """ + + use GenServer + + alias Explorer.Chain.Transaction + alias Explorer.Repo + + @tab :transaction_count_cache + # 2 hours + @cache_period 1_000 * 60 * 60 * 2 + @default_value 0 + @key "count" + @name __MODULE__ + + def start_link([params, gen_server_options]) do + GenServer.start_link(__MODULE__, params, name: gen_server_options[:name] || @name) + end + + def init(params) do + cache_period = params[:cache_period] || @cache_period + current_value = params[:default_value] || @default_value + + init_ets_table() + + schedule_cache_update() + + {:ok, {{cache_period, current_value}, nil}} + end + + def value(process_name \\ __MODULE__) do + GenServer.call(process_name, :value) + end + + def handle_call(:value, _, {{cache_period, default_value}, task}) do + {value, task} = + case cached_values() do + nil -> + {default_value, update_cache(task)} + + {cached_value, timestamp} -> + task = + if current_time() - timestamp > cache_period do + update_cache(task) + end + + {cached_value, task} + end + + {:reply, value, {{cache_period, default_value}, task}} + end + + def update_cache(nil) do + async_update_cache() + end + + def update_cache(task) do + task + end + + def handle_cast({:update_cache, value}, {{cache_period, default_value}, _}) do + current_time = current_time() + tuple = {value, current_time} + + :ets.insert(@tab, {@key, tuple}) + + {:noreply, {{cache_period, default_value}, nil}} + end + + def handle_info({:DOWN, _, _, _, _}, {{cache_period, default_value}, _}) do + {:noreply, {{cache_period, default_value}, nil}} + end + + def handle_info(_, {{cache_period, default_value}, _}) do + {:noreply, {{cache_period, default_value}, nil}} + end + + def async_update_cache do + Task.async(fn -> + result = Repo.aggregate(Transaction, :count, :hash, timeout: :infinity) + + GenServer.cast(__MODULE__, {:update_cache, result}) + end) + end + + defp init_ets_table do + if :ets.whereis(@tab) == :undefined do + :ets.new(@tab, [ + :set, + :named_table, + :public, + write_concurrency: true + ]) + end + end + + defp cached_values do + case :ets.lookup(@tab, @key) do + [{_, cached_values}] -> cached_values + _ -> nil + end + end + + defp schedule_cache_update do + Process.send_after(self(), :update_cache, 2_000) + end + + defp current_time do + utc_now = DateTime.utc_now() + + DateTime.to_unix(utc_now, :millisecond) + end +end diff --git a/apps/explorer/test/explorer/chain/transaction_count_cache_test.exs b/apps/explorer/test/explorer/chain/transaction_count_cache_test.exs new file mode 100644 index 0000000000..50c1e818a2 --- /dev/null +++ b/apps/explorer/test/explorer/chain/transaction_count_cache_test.exs @@ -0,0 +1,58 @@ +defmodule Explorer.Chain.TransactionCountCacheTest do + use Explorer.DataCase + + alias Explorer.Chain.TransactionCountCache + + test "returns default transaction count" do + TransactionCountCache.start_link([[], []]) + + result = TransactionCountCache.value() + + assert result == 0 + end + + test "updates cache if initial value is zero" do + TransactionCountCache.start_link([[], []]) + + insert(:transaction) + insert(:transaction) + + result = TransactionCountCache.value() + + assert result == 0 + + Process.sleep(500) + + updated_value = TransactionCountCache.value() + + assert updated_value == 2 + end + + test "does not update cache if cache period did not pass" do + TransactionCountCache.start_link([[], []]) + + insert(:transaction) + insert(:transaction) + + result = TransactionCountCache.value() + + assert result == 0 + + Process.sleep(500) + + updated_value = TransactionCountCache.value() + + assert updated_value == 2 + + insert(:transaction) + insert(:transaction) + + _updated_value = TransactionCountCache.value() + + Process.sleep(500) + + updated_value = TransactionCountCache.value() + + assert updated_value == 2 + end +end From 8a58d4b03eb138013293b952b1329661752e7d3f Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Fri, 22 Mar 2019 13:40:30 +0300 Subject: [PATCH 02/14] use cache for estimated transaction count --- apps/explorer/lib/explorer/application.ex | 4 +++- apps/explorer/lib/explorer/chain.ex | 7 ++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/apps/explorer/lib/explorer/application.ex b/apps/explorer/lib/explorer/application.ex index ecf4250213..b896e84a1f 100644 --- a/apps/explorer/lib/explorer/application.ex +++ b/apps/explorer/lib/explorer/application.ex @@ -8,6 +8,7 @@ defmodule Explorer.Application do alias Explorer.Admin alias Explorer.Chain.BlockNumberCache alias Explorer.Repo.PrometheusLogger + alias Explorer.Chain.TransactionCountCache @impl Application def start(_type, _args) do @@ -27,7 +28,8 @@ defmodule Explorer.Application do Supervisor.child_spec({Task.Supervisor, name: Explorer.MarketTaskSupervisor}, id: Explorer.MarketTaskSupervisor), Supervisor.child_spec({Task.Supervisor, name: Explorer.TaskSupervisor}, id: Explorer.TaskSupervisor), {Registry, keys: :duplicate, name: Registry.ChainEvents, id: Registry.ChainEvents}, - {Admin.Recovery, [[], [name: Admin.Recovery]]} + {Admin.Recovery, [[], [name: Admin.Recovery]]}, + {TransactionCountCache, [[], []]} ] children = base_children ++ configurable_children() diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index 58b6e1a34b..5bd6394606 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -20,7 +20,6 @@ defmodule Explorer.Chain do import EthereumJSONRPC, only: [integer_to_quantity: 1] - alias Ecto.Adapters.SQL alias Ecto.{Changeset, Multi} alias Explorer.Chain.{ @@ -39,6 +38,7 @@ defmodule Explorer.Chain do Token, TokenTransfer, Transaction, + TransactionCountCache, Wei } @@ -1842,10 +1842,7 @@ defmodule Explorer.Chain do """ @spec transaction_estimated_count() :: non_neg_integer() def transaction_estimated_count do - %Postgrex.Result{rows: [[rows]]} = - SQL.query!(Repo, "SELECT reltuples::BIGINT AS estimate FROM pg_class WHERE relname='transactions'") - - rows + TransactionCountCache.value() end @doc """ From 6360ecf7997d61023ee3eeaa43b28d88ed4a8768 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Fri, 22 Mar 2019 14:56:34 +0300 Subject: [PATCH 03/14] fix tests --- .../explorer/chain/transaction_count_cache.ex | 78 ++++++++++++------- .../chain/transaction_count_cache_test.exs | 30 ++++--- 2 files changed, 63 insertions(+), 45 deletions(-) diff --git a/apps/explorer/lib/explorer/chain/transaction_count_cache.ex b/apps/explorer/lib/explorer/chain/transaction_count_cache.ex index 9dc6be2e68..463a6d6cec 100644 --- a/apps/explorer/lib/explorer/chain/transaction_count_cache.ex +++ b/apps/explorer/lib/explorer/chain/transaction_count_cache.ex @@ -3,12 +3,13 @@ defmodule Explorer.Chain.TransactionCountCache do Cache for estimated transaction count. """ + require Logger + use GenServer alias Explorer.Chain.Transaction alias Explorer.Repo - @tab :transaction_count_cache # 2 hours @cache_period 1_000 * 60 * 60 * 2 @default_value 0 @@ -16,78 +17,97 @@ defmodule Explorer.Chain.TransactionCountCache do @name __MODULE__ def start_link([params, gen_server_options]) do - GenServer.start_link(__MODULE__, params, name: gen_server_options[:name] || @name) + name = gen_server_options[:name] || @name + params_with_name = Keyword.put(params, :name, name) + + GenServer.start_link(__MODULE__, params_with_name, name: name) end def init(params) do cache_period = params[:cache_period] || @cache_period current_value = params[:default_value] || @default_value + name = params[:name] - init_ets_table() + init_ets_table(name) schedule_cache_update() - {:ok, {{cache_period, current_value}, nil}} + {:ok, {{cache_period, current_value, name}, nil}} end def value(process_name \\ __MODULE__) do GenServer.call(process_name, :value) end - def handle_call(:value, _, {{cache_period, default_value}, task}) do + def handle_call(:value, _, {{cache_period, default_value, name}, task}) do {value, task} = - case cached_values() do + case cached_values(name) do nil -> - {default_value, update_cache(task)} + {default_value, update_cache(task, name)} {cached_value, timestamp} -> task = if current_time() - timestamp > cache_period do - update_cache(task) + update_cache(task, name) end {cached_value, task} end - {:reply, value, {{cache_period, default_value}, task}} + {:reply, value, {{cache_period, default_value, name}, task}} end - def update_cache(nil) do - async_update_cache() + def update_cache(nil, name) do + async_update_cache(name) end - def update_cache(task) do + def update_cache(task, _) do task end - def handle_cast({:update_cache, value}, {{cache_period, default_value}, _}) do + def handle_cast({:update_cache, value}, {{cache_period, default_value, name}, _}) do current_time = current_time() tuple = {value, current_time} - :ets.insert(@tab, {@key, tuple}) + table_name = table_name(name) + + :ets.insert(table_name, {@key, tuple}) - {:noreply, {{cache_period, default_value}, nil}} + {:noreply, {{cache_period, default_value, name}, nil}} end - def handle_info({:DOWN, _, _, _, _}, {{cache_period, default_value}, _}) do - {:noreply, {{cache_period, default_value}, nil}} + def handle_info({:DOWN, _, _, _, _}, {{cache_period, default_value, name}, _}) do + {:noreply, {{cache_period, default_value, name}, nil}} end - def handle_info(_, {{cache_period, default_value}, _}) do - {:noreply, {{cache_period, default_value}, nil}} + def handle_info(_, {{cache_period, default_value, name}, _}) do + {:noreply, {{cache_period, default_value, name}, nil}} end - def async_update_cache do - Task.async(fn -> - result = Repo.aggregate(Transaction, :count, :hash, timeout: :infinity) + defp table_name(name) do + name |> Atom.to_string() |> Macro.underscore() |> String.to_atom() + end - GenServer.cast(__MODULE__, {:update_cache, result}) + def async_update_cache(name) do + Task.async(fn -> + try do + result = Repo.aggregate(Transaction, :count, :hash, timeout: :infinity) + + GenServer.cast(name, {:update_cache, result}) + rescue + e -> + Logger.debug([ + "Coudn't update transaction count test #{inspect(e)}" + ]) + end end) end - defp init_ets_table do - if :ets.whereis(@tab) == :undefined do - :ets.new(@tab, [ + defp init_ets_table(name) do + table_name = table_name(name) + + if :ets.whereis(table_name) == :undefined do + :ets.new(table_name, [ :set, :named_table, :public, @@ -96,8 +116,10 @@ defmodule Explorer.Chain.TransactionCountCache do end end - defp cached_values do - case :ets.lookup(@tab, @key) do + defp cached_values(name) do + table_name = table_name(name) + + case :ets.lookup(table_name, @key) do [{_, cached_values}] -> cached_values _ -> nil end diff --git a/apps/explorer/test/explorer/chain/transaction_count_cache_test.exs b/apps/explorer/test/explorer/chain/transaction_count_cache_test.exs index 50c1e818a2..8ddcda2f04 100644 --- a/apps/explorer/test/explorer/chain/transaction_count_cache_test.exs +++ b/apps/explorer/test/explorer/chain/transaction_count_cache_test.exs @@ -4,54 +4,50 @@ defmodule Explorer.Chain.TransactionCountCacheTest do alias Explorer.Chain.TransactionCountCache test "returns default transaction count" do - TransactionCountCache.start_link([[], []]) + TransactionCountCache.start_link([[], [name: TestCache]]) - result = TransactionCountCache.value() + result = TransactionCountCache.value(TestCache) assert result == 0 end test "updates cache if initial value is zero" do - TransactionCountCache.start_link([[], []]) + TransactionCountCache.start_link([[], [name: TestCache]]) insert(:transaction) insert(:transaction) - result = TransactionCountCache.value() + _result = TransactionCountCache.value(TestCache) - assert result == 0 - - Process.sleep(500) + Process.sleep(1000) - updated_value = TransactionCountCache.value() + updated_value = TransactionCountCache.value(TestCache) assert updated_value == 2 end test "does not update cache if cache period did not pass" do - TransactionCountCache.start_link([[], []]) + TransactionCountCache.start_link([[], [name: TestCache]]) insert(:transaction) insert(:transaction) - result = TransactionCountCache.value() - - assert result == 0 + _result = TransactionCountCache.value(TestCache) - Process.sleep(500) + Process.sleep(1000) - updated_value = TransactionCountCache.value() + updated_value = TransactionCountCache.value(TestCache) assert updated_value == 2 insert(:transaction) insert(:transaction) - _updated_value = TransactionCountCache.value() + _updated_value = TransactionCountCache.value(TestCache) - Process.sleep(500) + Process.sleep(1000) - updated_value = TransactionCountCache.value() + updated_value = TransactionCountCache.value(TestCache) assert updated_value == 2 end From 6727a8bd5939c44f1d99e34f95325fcd1a8d0b03 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Mon, 25 Mar 2019 13:56:29 +0300 Subject: [PATCH 04/14] fix credo --- apps/explorer/lib/explorer/application.ex | 3 +-- apps/explorer/lib/explorer/chain/transaction_count_cache.ex | 6 +++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/apps/explorer/lib/explorer/application.ex b/apps/explorer/lib/explorer/application.ex index b896e84a1f..65f91c1d4b 100644 --- a/apps/explorer/lib/explorer/application.ex +++ b/apps/explorer/lib/explorer/application.ex @@ -6,9 +6,8 @@ defmodule Explorer.Application do use Application alias Explorer.Admin - alias Explorer.Chain.BlockNumberCache + alias Explorer.Chain.{BlockNumberCache, TransactionCountCache} alias Explorer.Repo.PrometheusLogger - alias Explorer.Chain.TransactionCountCache @impl Application def start(_type, _args) do diff --git a/apps/explorer/lib/explorer/chain/transaction_count_cache.ex b/apps/explorer/lib/explorer/chain/transaction_count_cache.ex index 463a6d6cec..809453338a 100644 --- a/apps/explorer/lib/explorer/chain/transaction_count_cache.ex +++ b/apps/explorer/lib/explorer/chain/transaction_count_cache.ex @@ -84,8 +84,12 @@ defmodule Explorer.Chain.TransactionCountCache do {:noreply, {{cache_period, default_value, name}, nil}} end + # sobelow_skip ["DOS"] defp table_name(name) do - name |> Atom.to_string() |> Macro.underscore() |> String.to_atom() + name + |> Atom.to_string() + |> Macro.underscore() + |> String.to_atom() end def async_update_cache(name) do From fd08bc1b3a188684469d58068c06ea728dfb3ada Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Mon, 25 Mar 2019 16:49:17 +0300 Subject: [PATCH 05/14] add CHANGELOG entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 684ed983e1..2f9e9e3a4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - [1611](https://github.com/poanetwork/blockscout/pull/1611) - allow setting the first indexing block - [1596](https://github.com/poanetwork/blockscout/pull/1596) - add endpoint to create decompiled contracts + - [1634](https://github.com/poanetwork/blockscout/pull/1634) - add transaction count cache ### Fixes From a6c4dce440be8d8e82d45e245f749d41160a26a7 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 26 Mar 2019 11:13:52 +0300 Subject: [PATCH 06/14] fetch transaction period from env variables --- apps/explorer/lib/explorer/chain/transaction_count_cache.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/explorer/lib/explorer/chain/transaction_count_cache.ex b/apps/explorer/lib/explorer/chain/transaction_count_cache.ex index 809453338a..884e1c7d1f 100644 --- a/apps/explorer/lib/explorer/chain/transaction_count_cache.ex +++ b/apps/explorer/lib/explorer/chain/transaction_count_cache.ex @@ -24,7 +24,7 @@ defmodule Explorer.Chain.TransactionCountCache do end def init(params) do - cache_period = params[:cache_period] || @cache_period + cache_period = System.get_env("TRANSACTION_COUNT_CACHE_PERIOD") || params[:cache_period] || @cache_period current_value = params[:default_value] || @default_value name = params[:name] From 62990e87cd49787ab6395994e73772af403ec3e0 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 26 Mar 2019 12:50:39 +0300 Subject: [PATCH 07/14] set only hours in env var --- .../explorer/lib/explorer/chain/transaction_count_cache.ex | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/apps/explorer/lib/explorer/chain/transaction_count_cache.ex b/apps/explorer/lib/explorer/chain/transaction_count_cache.ex index 884e1c7d1f..fa88cc5e9b 100644 --- a/apps/explorer/lib/explorer/chain/transaction_count_cache.ex +++ b/apps/explorer/lib/explorer/chain/transaction_count_cache.ex @@ -24,7 +24,7 @@ defmodule Explorer.Chain.TransactionCountCache do end def init(params) do - cache_period = System.get_env("TRANSACTION_COUNT_CACHE_PERIOD") || params[:cache_period] || @cache_period + cache_period = period_from_env_var() || params[:cache_period] || @cache_period current_value = params[:default_value] || @default_value name = params[:name] @@ -138,4 +138,9 @@ defmodule Explorer.Chain.TransactionCountCache do DateTime.to_unix(utc_now, :millisecond) end + + defp period_from_env_var do + env_var = System.get_env("TXS_COUNT_CACHE_PERIOD") + env_var && env_var * 1_000 * 60 * 60 + end end From dd9a0837bb28bc3dd2798399d7feb6fd08e371ff Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 26 Mar 2019 13:06:58 +0300 Subject: [PATCH 08/14] fix dialyzer --- .../lib/explorer/chain/transaction_count_cache.ex | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/apps/explorer/lib/explorer/chain/transaction_count_cache.ex b/apps/explorer/lib/explorer/chain/transaction_count_cache.ex index fa88cc5e9b..60ecc05285 100644 --- a/apps/explorer/lib/explorer/chain/transaction_count_cache.ex +++ b/apps/explorer/lib/explorer/chain/transaction_count_cache.ex @@ -140,7 +140,15 @@ defmodule Explorer.Chain.TransactionCountCache do end defp period_from_env_var do - env_var = System.get_env("TXS_COUNT_CACHE_PERIOD") - env_var && env_var * 1_000 * 60 * 60 + case System.get_env("TXS_COUNT_CACHE_PERIOD") do + value when is_binary(value) -> + case Integer.parse(value) do + {integer, ""} -> integer * 1_000 * 60 * 60 + _ -> nil + end + + _ -> + nil + end end end From e694d149782420029e095c51ee6e04e909f777b8 Mon Sep 17 00:00:00 2001 From: zachdaniel Date: Tue, 19 Mar 2019 15:00:30 -0400 Subject: [PATCH 09/14] feat: add listcontracts endpoint --- CHANGELOG.md | 1 + .../controllers/api/rpc/address_controller.ex | 23 +-- .../api/rpc/contract_controller.ex | 91 ++++++++- .../controllers/api/rpc/helpers.ex | 42 ++++ .../lib/block_scout_web/etherscan.ex | 117 ++++++++++- .../views/api/rpc/contract_view.ex | 96 ++++++--- .../api/rpc/address_controller_test.exs | 31 +-- .../api/rpc/contract_controller_test.exs | 192 +++++++++++++++++- apps/explorer/lib/explorer/chain.ex | 82 ++++++++ apps/explorer/lib/explorer/chain/address.ex | 15 +- .../lib/explorer/chain/smart_contract.ex | 13 +- 11 files changed, 618 insertions(+), 85 deletions(-) create mode 100644 apps/block_scout_web/lib/block_scout_web/controllers/api/rpc/helpers.ex diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b9b706ad9..06fa1d9937 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,7 @@ - [#1589](https://github.com/poanetwork/blockscout/pull/1589) - RPC endpoint to list addresses - [#1567](https://github.com/poanetwork/blockscout/pull/1567) - Allow setting different configuration just for realtime fetcher - [#1562](https://github.com/poanetwork/blockscout/pull/1562) - Add incoming transactions count to contract view + - [#1608](https://github.com/poanetwork/blockscout/pull/1608) - Add listcontracts RPC Endpoint ### Fixes diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/api/rpc/address_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/api/rpc/address_controller.ex index 76ddf4342c..c2bbfd5317 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/api/rpc/address_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/api/rpc/address_controller.ex @@ -1,6 +1,7 @@ defmodule BlockScoutWeb.API.RPC.AddressController do use BlockScoutWeb, :controller + alias BlockScoutWeb.API.RPC.Helpers alias Explorer.{Chain, Etherscan} alias Explorer.Chain.{Address, Wei} @@ -162,7 +163,7 @@ defmodule BlockScoutWeb.API.RPC.AddressController do end def getminedblocks(conn, params) do - options = put_pagination_options(%{}, params) + options = Helpers.put_pagination_options(%{}, params) with {:address_param, {:ok, address_param}} <- fetch_address(params), {:format, {:ok, address_hash}} <- to_address_hash(address_param), @@ -188,7 +189,7 @@ defmodule BlockScoutWeb.API.RPC.AddressController do def optional_params(params) do %{} |> put_order_by_direction(params) - |> put_pagination_options(params) + |> Helpers.put_pagination_options(params) |> put_start_block(params) |> put_end_block(params) |> put_filter_by(params) @@ -338,24 +339,6 @@ defmodule BlockScoutWeb.API.RPC.AddressController do end end - defp put_pagination_options(options, params) do - with %{"page" => page, "offset" => offset} <- params, - {page_number, ""} when page_number > 0 <- Integer.parse(page), - {page_size, ""} when page_size > 0 <- Integer.parse(offset), - :ok <- validate_max_page_size(page_size) do - options - |> Map.put(:page_number, page_number) - |> Map.put(:page_size, page_size) - else - _ -> - options - end - end - - defp validate_max_page_size(page_size) do - if page_size <= Etherscan.page_size_max(), do: :ok, else: :error - end - defp put_start_block(options, params) do with %{"startblock" => startblock_param} <- params, {start_block, ""} <- Integer.parse(startblock_param) do diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/api/rpc/contract_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/api/rpc/contract_controller.ex index 5a3a44b799..8a8036f2b4 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/api/rpc/contract_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/api/rpc/contract_controller.ex @@ -1,7 +1,30 @@ defmodule BlockScoutWeb.API.RPC.ContractController do use BlockScoutWeb, :controller + alias BlockScoutWeb.API.RPC.Helpers alias Explorer.Chain + alias Explorer.Chain.SmartContract + + def listcontracts(conn, params) do + with pagination_options <- Helpers.put_pagination_options(%{}, params), + {:params, {:ok, options}} <- {:params, add_filter(pagination_options, params)} do + options_with_defaults = + options + |> Map.put_new(:page_number, 0) + |> Map.put_new(:page_size, 10) + + contracts = list_contracts(options_with_defaults) + + conn + |> put_status(200) + |> render(:listcontracts, %{contracts: contracts}) + else + {:params, {:error, error}} -> + conn + |> put_status(400) + |> render(:error, error: error) + end + end def getabi(conn, params) do with {:address_param, {:ok, address_param}} <- fetch_address(params), @@ -24,7 +47,10 @@ defmodule BlockScoutWeb.API.RPC.ContractController do with {:address_param, {:ok, address_param}} <- fetch_address(params), {:format, {:ok, address_hash}} <- to_address_hash(address_param), {:contract, {:ok, contract}} <- to_smart_contract(address_hash) do - render(conn, :getsourcecode, %{contract: contract}) + render(conn, :getsourcecode, %{ + contract: contract, + address_hash: address_hash + }) else {:address_param, :error} -> render(conn, :error, error: "Query parameter address is required") @@ -33,10 +59,64 @@ defmodule BlockScoutWeb.API.RPC.ContractController do render(conn, :error, error: "Invalid address hash") {:contract, :not_found} -> - render(conn, :getsourcecode, %{contract: nil}) + render(conn, :getsourcecode, %{contract: nil, address_hash: nil}) + end + end + + defp list_contracts(%{page_number: page_number, page_size: page_size} = opts) do + offset = (max(page_number, 1) - 1) * page_size + + case Map.get(opts, :filter) do + :verified -> + Chain.list_verified_contracts(page_size, offset) + + :decompiled -> + Chain.list_decompiled_contracts(page_size, offset) + + :unverified -> + Chain.list_unverified_contracts(page_size, offset) + + :not_decompiled -> + Chain.list_not_decompiled_contracts(page_size, offset) + + _ -> + Chain.list_contracts(page_size, offset) end end + defp add_filter(options, params) do + with {:param, {:ok, value}} <- {:param, Map.fetch(params, "filter")}, + {:validation, {:ok, filter}} <- {:validation, contracts_filter(value)} do + {:ok, Map.put(options, :filter, filter)} + else + {:param, :error} -> {:ok, options} + {:validation, {:error, error}} -> {:error, error} + end + end + + defp contracts_filter(nil), do: {:ok, nil} + defp contracts_filter(1), do: {:ok, :verified} + defp contracts_filter(2), do: {:ok, :decompiled} + defp contracts_filter(3), do: {:ok, :unverified} + defp contracts_filter(4), do: {:ok, :not_decompiled} + defp contracts_filter("verified"), do: {:ok, :verified} + defp contracts_filter("decompiled"), do: {:ok, :decompiled} + defp contracts_filter("unverified"), do: {:ok, :unverified} + defp contracts_filter("not_decompiled"), do: {:ok, :not_decompiled} + + defp contracts_filter(filter) when is_bitstring(filter) do + case Integer.parse(filter) do + {number, ""} -> contracts_filter(number) + _ -> {:error, contracts_filter_error_message(filter)} + end + end + + defp contracts_filter(filter), do: {:error, contracts_filter_error_message(filter)} + + defp contracts_filter_error_message(filter) do + "#{filter} is not a valid value for `filter`. Please use one of: verified, decompiled, unverified, not_decompiled, 1, 2, 3, 4." + end + defp fetch_address(params) do {:address_param, Map.fetch(params, "address")} end @@ -48,8 +128,11 @@ defmodule BlockScoutWeb.API.RPC.ContractController do defp to_smart_contract(address_hash) do result = case Chain.address_hash_to_smart_contract(address_hash) do - nil -> :not_found - contract -> {:ok, contract} + nil -> + :not_found + + contract -> + {:ok, SmartContract.preload_decompiled_smart_contract(contract)} end {:contract, result} diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/api/rpc/helpers.ex b/apps/block_scout_web/lib/block_scout_web/controllers/api/rpc/helpers.ex new file mode 100644 index 0000000000..1196af9edf --- /dev/null +++ b/apps/block_scout_web/lib/block_scout_web/controllers/api/rpc/helpers.ex @@ -0,0 +1,42 @@ +defmodule BlockScoutWeb.API.RPC.Helpers do + @moduledoc """ + Small helpers for RPC api controllers. + """ + alias Explorer.Etherscan + + def put_pagination_options(options, params) do + options + |> put_page_option(params) + |> put_offset_option(params) + end + + def put_page_option(options, %{"page" => page}) do + case Integer.parse(page) do + {page_number, ""} when page_number > 0 -> + Map.put(options, :page_number, page_number) + + _ -> + options + end + end + + def put_page_option(options, _), do: options + + def put_offset_option(options, %{"offset" => offset}) do + with {page_size, ""} when page_size > 0 <- Integer.parse(offset), + :ok <- validate_max_page_size(page_size) do + Map.put(options, :page_size, page_size) + else + _ -> + options + end + end + + def put_offset_option(options, _) do + options + end + + defp validate_max_page_size(page_size) do + if page_size <= Etherscan.page_size_max(), do: :ok, else: :error + end +end diff --git a/apps/block_scout_web/lib/block_scout_web/etherscan.ex b/apps/block_scout_web/lib/block_scout_web/etherscan.ex index e2282f74fe..4162a77e70 100644 --- a/apps/block_scout_web/lib/block_scout_web/etherscan.ex +++ b/apps/block_scout_web/lib/block_scout_web/etherscan.ex @@ -276,6 +276,45 @@ defmodule BlockScoutWeb.Etherscan do "result" => nil } + @contract_listcontracts_example_value %{ + "status" => "1", + "message" => "OK", + "result" => [ + %{ + "SourceCode" => """ + pragma solidity >0.4.24; + + contract Test { + constructor() public { b = hex"12345678901234567890123456789012"; } + event Event(uint indexed a, bytes32 b); + event Event2(uint indexed a, bytes32 b); + function foo(uint a) public { emit Event(a, b); } + bytes32 b; + } + """, + "ABI" => """ + [{ + "type":"event", + "inputs": [{"name":"a","type":"uint256","indexed":true},{"name":"b","type":"bytes32","indexed":false}], + "name":"Event" + }, { + "type":"event", + "inputs": [{"name":"a","type":"uint256","indexed":true},{"name":"b","type":"bytes32","indexed":false}], + "name":"Event2" + }, { + "type":"function", + "inputs": [{"name":"a","type":"uint256"}], + "name":"foo", + "outputs": [] + }] + """, + "ContractName" => "Test", + "CompilerVersion" => "v0.2.1-2016-01-30-91a6b35", + "OptimizationUsed" => "1" + } + ] + } + @contract_getabi_example_value %{ "status" => "1", "message" => "OK", @@ -742,6 +781,7 @@ defmodule BlockScoutWeb.Etherscan do @contract_model %{ name: "Contract", fields: %{ + "Address" => @address_hash_type, "SourceCode" => %{ type: "contract source code", definition: "The contract's source code.", @@ -757,6 +797,33 @@ defmodule BlockScoutWeb.Etherscan do }" """ }, + "DecompilerVersion" => %{ + type: "decompiler version", + definition: "When decompiled source code is present, the decompiler version with which it was generated.", + example: "decompiler.version" + }, + "DecompiledSourceCode" => %{ + type: "contract decompiled source code", + definition: "The contract's decompiled source code.", + example: """ + const name() = 'CryptoKitties' + const GEN0_STARTING_PRICE() = 10^16 + const GEN0_AUCTION_DURATION() = 86400 + const GEN0_CREATION_LIMIT() = 45000 + const symbol() = 'CK' + const PROMO_CREATION_LIMIT() = 5000 + def storage: + ceoAddress is addr # mask(160, 0) at storage #0 + cfoAddress is addr # mask(160, 0) at storage #1 + stor1.768 is uint16 => uint256 # mask(256, 768) at storage #1 + cooAddress is addr # mask(160, 0) at storage #2 + stor2.0 is uint256 => uint256 # mask(256, 0) at storage #2 + paused is uint8 # mask(8, 160) at storage #2 + stor2.256 is uint256 => uint256 # mask(256, 256) at storage #2 + stor3 is uint32 # + ... + """ + }, "ABI" => %{ type: "ABI", definition: "JSON string for the contract's Application Binary Interface (ABI)", @@ -1343,11 +1410,6 @@ defmodule BlockScoutWeb.Etherscan do } } } - }, - %{ - code: "200", - description: "error", - example_value: Jason.encode!(@account_getminedblocks_example_value_error) } ] } @@ -1635,6 +1697,50 @@ defmodule BlockScoutWeb.Etherscan do ] } + @contract_listcontracts_action %{ + name: "listcontracts", + description: "Get a list of contracts, sorted ascending by the time they were first seen by the explorer.", + required_params: [], + optional_params: [ + %{ + key: "page", + type: "integer", + description: + "A nonnegative integer that represents the page number to be used for pagination. 'offset' must be provided in conjunction." + }, + %{ + key: "offset", + type: "integer", + description: + "A nonnegative integer that represents the maximum number of records to return when paginating. 'page' must be provided in conjunction." + }, + %{ + key: "filter", + type: "string", + description: + "verified|decompiled|unverified|not_decompiled, or 1|2|3|4 respectively. This requests only contracts with that status." + } + ], + responses: [ + %{ + code: "200", + description: "successful operation", + example_value: Jason.encode!(@contract_listcontracts_example_value), + model: %{ + name: "Result", + fields: %{ + status: @status_type, + message: @message_type, + result: %{ + type: "array", + array_type: @contract_model + } + } + } + } + ] + } + @contract_getabi_action %{ name: "getabi", description: "Get ABI for verified contract. Also available through a GraphQL 'addresses' query.", @@ -1862,6 +1968,7 @@ defmodule BlockScoutWeb.Etherscan do @contract_module %{ name: "contract", actions: [ + @contract_listcontracts_action, @contract_getabi_action, @contract_getsourcecode_action ] diff --git a/apps/block_scout_web/lib/block_scout_web/views/api/rpc/contract_view.ex b/apps/block_scout_web/lib/block_scout_web/views/api/rpc/contract_view.ex index 322df4257d..fbe1813dce 100644 --- a/apps/block_scout_web/lib/block_scout_web/views/api/rpc/contract_view.ex +++ b/apps/block_scout_web/lib/block_scout_web/views/api/rpc/contract_view.ex @@ -2,40 +2,88 @@ defmodule BlockScoutWeb.API.RPC.ContractView do use BlockScoutWeb, :view alias BlockScoutWeb.API.RPC.RPCView + alias Explorer.Chain.{Address, DecompiledSmartContract, SmartContract} + + def render("listcontracts.json", %{contracts: contracts}) do + contracts = Enum.map(contracts, &prepare_contract/1) + + RPCView.render("show.json", data: contracts) + end def render("getabi.json", %{abi: abi}) do RPCView.render("show.json", data: Jason.encode!(abi)) end - def render("getsourcecode.json", %{contract: contract}) do - RPCView.render("show.json", data: prepare_contract(contract)) + def render("getsourcecode.json", %{contract: contract, address_hash: address_hash}) do + RPCView.render("show.json", data: [prepare_source_code_contract(contract, address_hash)]) end def render("error.json", assigns) do RPCView.render("error.json", assigns) end - defp prepare_contract(nil) do - [ - %{ - "SourceCode" => "", - "ABI" => "Contract source code not verified", - "ContractName" => "", - "CompilerVersion" => "", - "OptimizationUsed" => "" - } - ] - end - - defp prepare_contract(contract) do - [ - %{ - "SourceCode" => contract.contract_source_code, - "ABI" => Jason.encode!(contract.abi), - "ContractName" => contract.name, - "CompilerVersion" => contract.compiler_version, - "OptimizationUsed" => if(contract.optimization, do: "1", else: "0") - } - ] + defp prepare_source_code_contract(nil, address_hash) do + %{ + "Address" => to_string(address_hash), + "SourceCode" => "", + "ABI" => "Contract source code not verified", + "ContractName" => "", + "CompilerVersion" => "", + "DecompiledSourceCode" => "", + "DecompilerVersion" => "", + "OptimizationUsed" => "" + } + end + + defp prepare_source_code_contract(contract, _) do + %{ + "Address" => to_string(contract.address_hash), + "SourceCode" => contract.contract_source_code, + "ABI" => Jason.encode!(contract.abi), + "ContractName" => contract.name, + "DecompiledSourceCode" => decompiled_source_code(contract.decompiled_smart_contract), + "DecompilerVersion" => decompiler_version(contract.decompiled_smart_contract), + "CompilerVersion" => contract.compiler_version, + "OptimizationUsed" => if(contract.optimization, do: "1", else: "0") + } + end + + defp prepare_contract(%Address{hash: hash, smart_contract: nil, decompiled_smart_contract: decompiled_smart_contract}) do + %{ + "Address" => to_string(hash), + "SourceCode" => "", + "ABI" => "Contract source code not verified", + "ContractName" => "", + "DecompiledSourceCode" => decompiled_source_code(decompiled_smart_contract), + "DecompilerVersion" => decompiler_version(decompiled_smart_contract), + "CompilerVersion" => "", + "OptimizationUsed" => "" + } end + + defp prepare_contract(%Address{ + hash: hash, + smart_contract: %SmartContract{} = contract, + decompiled_smart_contract: decompiled_smart_contract + }) do + %{ + "Address" => to_string(hash), + "SourceCode" => contract.contract_source_code, + "ABI" => Jason.encode!(contract.abi), + "ContractName" => contract.name, + "DecompiledSourceCode" => decompiled_source_code(decompiled_smart_contract), + "DecompilerVersion" => decompiler_version(decompiled_smart_contract), + "CompilerVersion" => contract.compiler_version, + "OptimizationUsed" => if(contract.optimization, do: "1", else: "0") + } + end + + defp decompiled_source_code(nil), do: "Contract source code not decompiled." + + defp decompiled_source_code(%DecompiledSmartContract{decompiled_source_code: decompiled_source_code}) do + decompiled_source_code + end + + defp decompiler_version(nil), do: "" + defp decompiler_version(%DecompiledSmartContract{decompiler_version: decompiler_version}), do: decompiler_version end diff --git a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/address_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/address_controller_test.exs index 30cdbd84b0..6e0eccc7c0 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/address_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/address_controller_test.exs @@ -767,34 +767,7 @@ defmodule BlockScoutWeb.API.RPC.AddressControllerTest do assert response["message"] == "OK" end - test "ignores pagination params if page is less than 1", %{conn: conn} do - address = insert(:address) - - 6 - |> insert_list(:transaction, from_address: address) - |> with_block() - - params = %{ - "module" => "account", - "action" => "txlist", - "address" => "#{address.hash}", - # page number - "page" => "0", - # page size - "offset" => "2" - } - - assert response = - conn - |> get("/api", params) - |> json_response(200) - - assert length(response["result"]) == 6 - assert response["status"] == "1" - assert response["message"] == "OK" - end - - test "ignores pagination params if offset is less than 1", %{conn: conn} do + test "ignores offset param if offset is less than 1", %{conn: conn} do address = insert(:address) 6 @@ -821,7 +794,7 @@ defmodule BlockScoutWeb.API.RPC.AddressControllerTest do assert response["message"] == "OK" end - test "ignores pagination params if offset is over 10,000", %{conn: conn} do + test "ignores offset param if offset is over 10,000", %{conn: conn} do address = insert(:address) 6 diff --git a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/contract_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/contract_controller_test.exs index bd6a322c7b..766a691a62 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/contract_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/contract_controller_test.exs @@ -1,6 +1,190 @@ defmodule BlockScoutWeb.API.RPC.ContractControllerTest do use BlockScoutWeb.ConnCase + describe "listcontracts" do + setup do + %{params: %{"module" => "contract", "action" => "listcontracts"}} + end + + test "with an invalid filter value", %{conn: conn, params: params} do + response = + conn + |> get("/api", Map.put(params, "filter", "invalid")) + |> json_response(400) + + assert response["message"] == + "invalid is not a valid value for `filter`. Please use one of: verified, decompiled, unverified, not_decompiled, 1, 2, 3, 4." + + assert response["status"] == "0" + end + + test "with no contracts", %{conn: conn, params: params} do + response = + conn + |> get("/api", params) + |> json_response(200) + + assert response["message"] == "OK" + assert response["status"] == "1" + assert response["result"] == [] + end + + test "with a verified smart contract, all contract information is shown", %{conn: conn, params: params} do + contract = insert(:smart_contract) + + response = + conn + |> get("/api", params) + |> json_response(200) + + assert response["message"] == "OK" + assert response["status"] == "1" + + assert response["result"] == [ + %{ + "ABI" => Jason.encode!(contract.abi), + "Address" => to_string(contract.address_hash), + "CompilerVersion" => contract.compiler_version, + "ContractName" => contract.name, + "DecompiledSourceCode" => "Contract source code not decompiled.", + "DecompilerVersion" => "", + "OptimizationUsed" => if(contract.optimization, do: "1", else: "0"), + "SourceCode" => contract.contract_source_code + } + ] + end + + test "with an unverified contract address, only basic information is shown", %{conn: conn, params: params} do + address = insert(:contract_address) + + response = + conn + |> get("/api", params) + |> json_response(200) + + assert response["message"] == "OK" + assert response["status"] == "1" + + assert response["result"] == [ + %{ + "ABI" => "Contract source code not verified", + "Address" => to_string(address.hash), + "CompilerVersion" => "", + "ContractName" => "", + "DecompiledSourceCode" => "Contract source code not decompiled.", + "DecompilerVersion" => "", + "OptimizationUsed" => "", + "SourceCode" => "" + } + ] + end + + test "filtering for only unverified contracts shows only unverified contracts", %{params: params, conn: conn} do + address = insert(:contract_address) + insert(:smart_contract) + + response = + conn + |> get("/api", Map.put(params, "filter", "unverified")) + |> json_response(200) + + assert response["message"] == "OK" + assert response["status"] == "1" + + assert response["result"] == [ + %{ + "ABI" => "Contract source code not verified", + "Address" => to_string(address.hash), + "CompilerVersion" => "", + "ContractName" => "", + "DecompiledSourceCode" => "Contract source code not decompiled.", + "DecompilerVersion" => "", + "OptimizationUsed" => "", + "SourceCode" => "" + } + ] + end + + test "filtering for only verified contracts shows only verified contracts", %{params: params, conn: conn} do + insert(:contract_address) + contract = insert(:smart_contract) + + response = + conn + |> get("/api", Map.put(params, "filter", "verified")) + |> json_response(200) + + assert response["message"] == "OK" + assert response["status"] == "1" + + assert response["result"] == [ + %{ + "ABI" => Jason.encode!(contract.abi), + "Address" => to_string(contract.address_hash), + "CompilerVersion" => contract.compiler_version, + "DecompiledSourceCode" => "Contract source code not decompiled.", + "DecompilerVersion" => "", + "ContractName" => contract.name, + "OptimizationUsed" => if(contract.optimization, do: "1", else: "0"), + "SourceCode" => contract.contract_source_code + } + ] + end + + test "filtering for only decompiled contracts shows only decompiled contracts", %{params: params, conn: conn} do + insert(:contract_address) + decompiled_smart_contract = insert(:decompiled_smart_contract) + + response = + conn + |> get("/api", Map.put(params, "filter", "decompiled")) + |> json_response(200) + + assert response["message"] == "OK" + assert response["status"] == "1" + + assert response["result"] == [ + %{ + "ABI" => "Contract source code not verified", + "Address" => to_string(decompiled_smart_contract.address_hash), + "CompilerVersion" => "", + "ContractName" => "", + "DecompiledSourceCode" => decompiled_smart_contract.decompiled_source_code, + "DecompilerVersion" => "test_decompiler", + "OptimizationUsed" => "", + "SourceCode" => "" + } + ] + end + + test "filtering for only not_decompiled (and by extension not verified contracts)", %{params: params, conn: conn} do + insert(:decompiled_smart_contract) + insert(:smart_contract) + contract_address = insert(:contract_address) + + response = + conn + |> get("/api", Map.put(params, "filter", "not_decompiled")) + |> json_response(200) + + assert response["message"] == "OK" + assert response["status"] == "1" + + assert response["result"] == [ + %{ + "ABI" => "Contract source code not verified", + "Address" => to_string(contract_address.hash), + "CompilerVersion" => "", + "ContractName" => "", + "DecompiledSourceCode" => "Contract source code not decompiled.", + "DecompilerVersion" => "", + "OptimizationUsed" => "", + "SourceCode" => "" + } + ] + end + end + describe "getabi" do test "with missing address hash", %{conn: conn} do params = %{ @@ -119,11 +303,14 @@ defmodule BlockScoutWeb.API.RPC.ContractControllerTest do expected_result = [ %{ + "Address" => "", "SourceCode" => "", "ABI" => "Contract source code not verified", "ContractName" => "", "CompilerVersion" => "", - "OptimizationUsed" => "" + "OptimizationUsed" => "", + "DecompiledSourceCode" => "", + "DecompilerVersion" => "" } ] @@ -148,13 +335,16 @@ defmodule BlockScoutWeb.API.RPC.ContractControllerTest do expected_result = [ %{ + "Address" => to_string(contract.address_hash), "SourceCode" => contract.contract_source_code, "ABI" => Jason.encode!(contract.abi), "ContractName" => contract.name, "CompilerVersion" => contract.compiler_version, + "DecompiledSourceCode" => "Contract source code not decompiled.", # The contract's optimization value is true, so the expected value # for `OptimizationUsed` is "1". If it was false, the expected value # would be "0". + "DecompilerVersion" => "", "OptimizationUsed" => "1" } ] diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index 5860f80eb9..f2f594af94 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -2629,6 +2629,88 @@ defmodule Explorer.Chain do Repo.all(query, timeout: :infinity) end + def list_decompiled_contracts(limit, offset) do + query = + from( + address in Address, + join: decompiled_smart_contract in DecompiledSmartContract, + on: decompiled_smart_contract.address_hash == address.hash, + preload: [{:decompiled_smart_contract, decompiled_smart_contract}, :smart_contract], + order_by: [asc: address.inserted_at], + limit: ^limit, + offset: ^offset + ) + + Repo.all(query) + end + + def list_verified_contracts(limit, offset) do + query = + from( + address in Address, + where: not is_nil(address.contract_code), + join: smart_contract in SmartContract, + on: smart_contract.address_hash == address.hash, + preload: [{:smart_contract, smart_contract}, :decompiled_smart_contract], + order_by: [asc: address.inserted_at], + limit: ^limit, + offset: ^offset + ) + + Repo.all(query) + end + + def list_contracts(limit, offset) do + query = + from( + address in Address, + where: not is_nil(address.contract_code), + preload: [:smart_contract, :decompiled_smart_contract], + order_by: [asc: address.inserted_at], + limit: ^limit, + offset: ^offset + ) + + Repo.all(query) + end + + def list_unverified_contracts(limit, offset) do + query = + from( + address in Address, + left_join: smart_contract in SmartContract, + on: smart_contract.address_hash == address.hash, + where: not is_nil(address.contract_code), + where: is_nil(smart_contract.address_hash), + preload: [{:smart_contract, smart_contract}, :decompiled_smart_contract], + order_by: [asc: address.inserted_at], + limit: ^limit, + offset: ^offset + ) + + Repo.all(query) + end + + def list_not_decompiled_contracts(limit, offset) do + query = + from( + address in Address, + left_join: smart_contract in SmartContract, + on: smart_contract.address_hash == address.hash, + left_join: decompiled_smart_contract in DecompiledSmartContract, + on: decompiled_smart_contract.address_hash == address.hash, + preload: [smart_contract: smart_contract, decompiled_smart_contract: decompiled_smart_contract], + where: not is_nil(address.contract_code), + where: is_nil(smart_contract.address_hash), + where: is_nil(decompiled_smart_contract.address_hash), + order_by: [asc: address.inserted_at], + limit: ^limit, + offset: ^offset + ) + + Repo.all(query) + end + @doc """ Combined block reward from all the fees. """ diff --git a/apps/explorer/lib/explorer/chain/address.ex b/apps/explorer/lib/explorer/chain/address.ex index ee1c09eba2..7d9915d4df 100644 --- a/apps/explorer/lib/explorer/chain/address.ex +++ b/apps/explorer/lib/explorer/chain/address.ex @@ -8,7 +8,19 @@ defmodule Explorer.Chain.Address do use Explorer.Schema alias Ecto.Changeset - alias Explorer.Chain.{Address, Block, Data, Hash, InternalTransaction, SmartContract, Token, Transaction, Wei} + + alias Explorer.Chain.{ + Address, + Block, + Data, + DecompiledSmartContract, + Hash, + InternalTransaction, + SmartContract, + Token, + Transaction, + Wei + } @optional_attrs ~w(contract_code fetched_coin_balance fetched_coin_balance_block_number nonce)a @required_attrs ~w(hash)a @@ -64,6 +76,7 @@ defmodule Explorer.Chain.Address do field(:nonce, :integer) has_one(:smart_contract, SmartContract) + has_one(:decompiled_smart_contract, DecompiledSmartContract) has_one(:token, Token, foreign_key: :contract_address_hash) has_one( diff --git a/apps/explorer/lib/explorer/chain/smart_contract.ex b/apps/explorer/lib/explorer/chain/smart_contract.ex index dcdad6025c..a117a77353 100644 --- a/apps/explorer/lib/explorer/chain/smart_contract.ex +++ b/apps/explorer/lib/explorer/chain/smart_contract.ex @@ -12,7 +12,8 @@ defmodule Explorer.Chain.SmartContract do use Explorer.Schema - alias Explorer.Chain.{Address, ContractMethod, Hash} + alias Explorer.Chain.{Address, ContractMethod, DecompiledSmartContract, Hash} + alias Explorer.Repo @typedoc """ The name of a parameter to a function or event. @@ -208,6 +209,12 @@ defmodule Explorer.Chain.SmartContract do field(:constructor_arguments, :string) field(:abi, {:array, :map}) + has_one( + :decompiled_smart_contract, + DecompiledSmartContract, + foreign_key: :address_hash + ) + belongs_to( :address, Address, @@ -219,6 +226,10 @@ defmodule Explorer.Chain.SmartContract do timestamps() end + def preload_decompiled_smart_contract(contract) do + Repo.preload(contract, :decompiled_smart_contract) + end + def changeset(%__MODULE__{} = smart_contract, attrs) do smart_contract |> cast(attrs, [ From 5f7047edc8360a2a6f6832aad8e3d0bc2263df15 Mon Sep 17 00:00:00 2001 From: goodsoft Date: Tue, 26 Mar 2019 22:24:09 +0200 Subject: [PATCH 10/14] Add regression test for #1644 --- .../chain/import/runner/blocks_test.exs | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs b/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs index be882c2f7b..3d6fd912cc 100644 --- a/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs +++ b/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs @@ -8,6 +8,7 @@ defmodule Explorer.Chain.Import.Runner.BlocksTest do alias Ecto.Multi alias Explorer.Chain.Import.Runner.{Blocks, Transaction} alias Explorer.Chain.{Address, Block, Transaction} + alias Explorer.Chain alias Explorer.Repo describe "run/1" do @@ -258,6 +259,36 @@ defmodule Explorer.Chain.Import.Runner.BlocksTest do blocks_update_token_holder_counts: [] }} = run_block_consensus_change(block, true, options) end + + # Regression test for https://github.com/poanetwork/blockscout/issues/1644 + test "discards parent block if it isn't related to the current one because of reorg", + %{consensus_block: %Block{number: block_number, hash: block_hash, miner_hash: miner_hash}, options: options} do + old_block = insert(:block, parent_hash: block_hash, number: block_number + 1) + insert(:block, parent_hash: old_block.hash, number: old_block.number + 1) + + new_block1 = params_for(:block, parent_hash: block_hash, number: block_number + 1, miner_hash: miner_hash) + + new_block2 = + params_for(:block, parent_hash: new_block1.hash, number: new_block1.number + 1, miner_hash: miner_hash) + + %Ecto.Changeset{valid?: true, changes: block_changes} = Block.changeset(%Block{}, new_block2) + changes_list = [block_changes] + + Multi.new() + |> Blocks.run(changes_list, options) + |> Repo.transaction() + + assert Chain.missing_block_number_ranges(block_number..new_block2.number) == [old_block.number..old_block.number] + + %Ecto.Changeset{valid?: true, changes: block_changes} = Block.changeset(%Block{}, new_block1) + changes_list = [block_changes] + + Multi.new() + |> Blocks.run(changes_list, options) + |> Repo.transaction() + + assert Chain.missing_block_number_ranges(block_number..new_block2.number) == [] + end end defp count(schema) do From e1d13a3d7fd99ec3f77892e885a1f3fdc208d616 Mon Sep 17 00:00:00 2001 From: goodsoft Date: Tue, 26 Mar 2019 22:28:33 +0200 Subject: [PATCH 11/14] Force consensus loss for parent block if its hash mismatches parent_hash Fixes #1644 When reorg occurs and the older of new blocks fails to be imported, the old consensus block remains in the database. Catchup fetcher ignores it, and we get a discontinuity in the main chain. This commit adds an additional consensus loss step during block import: the parent block with hash not matching parent_hash of current block is marked non-consensus, thus creating a hole in main chain, forcing catchup fetcher to retry fetching it. --- CHANGELOG.md | 1 + .../explorer/chain/import/runner/blocks.ex | 44 +++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6503bb8064..2837e4210b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ - [#1643](https://github.com/poanetwork/blockscout/pull/1643) - Set internal_transactions_indexed_at for empty blocks - [#1647](https://github.com/poanetwork/blockscout/pull/1647) - Fix typo in view - [#1650](https://github.com/poanetwork/blockscout/pull/1650) - Add petersburg evm version to smart contract verifier + - [#1657](https://github.com/poanetwork/blockscout/pull/1657) - Force consensus loss for parent block if its hash mismatches parent_hash ### Chore diff --git a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex index e9b05bc4c0..c351bfd218 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex @@ -46,6 +46,7 @@ defmodule Explorer.Chain.Import.Runner.Blocks do |> Map.put(:timestamps, timestamps) ordered_consensus_block_numbers = ordered_consensus_block_numbers(changes_list) + where_invalid_parent = where_invalid_parent(changes_list) where_forked = where_forked(changes_list) multi @@ -69,6 +70,9 @@ defmodule Explorer.Chain.Import.Runner.Blocks do |> Multi.run(:lose_consensus, fn repo, _ -> lose_consensus(repo, ordered_consensus_block_numbers, insert_options) end) + |> Multi.run(:lose_invalid_parent_consensus, fn repo, _ -> + lose_invalid_parent_consensus(repo, where_invalid_parent, insert_options) + end) |> Multi.run(:delete_address_token_balances, fn repo, _ -> delete_address_token_balances(repo, ordered_consensus_block_numbers, insert_options) end) @@ -312,6 +316,32 @@ defmodule Explorer.Chain.Import.Runner.Blocks do end end + defp lose_invalid_parent_consensus(repo, where_invalid_parent, %{ + timeout: timeout, + timestamps: %{updated_at: updated_at} + }) do + query = + from( + block in where_invalid_parent, + update: [ + set: [ + consensus: false, + updated_at: ^updated_at + ] + ], + select: [:hash, :number] + ) + + try do + {_, result} = repo.update_all(query, [], timeout: timeout) + + {:ok, result} + rescue + postgrex_error in Postgrex.Error -> + {:error, %{exception: postgrex_error, where_invalid_parent: where_invalid_parent}} + end + end + defp delete_address_token_balances(_, [], _), do: {:ok, []} defp delete_address_token_balances(repo, ordered_consensus_block_numbers, %{timeout: timeout}) do @@ -552,4 +582,18 @@ defmodule Explorer.Chain.Import.Runner.Blocks do end end) end + + defp where_invalid_parent(blocks_changes) when is_list(blocks_changes) do + initial = from(b in Block, where: false) + + Enum.reduce(blocks_changes, initial, fn %{consensus: consensus, parent_hash: parent_hash, number: number}, acc -> + case consensus do + false -> + acc + + true -> + from(block in acc, or_where: block.number == ^(number - 1) and block.hash != ^parent_hash) + end + end) + end end From fe25bf7b44e45417fc57bcda008afc86e35e03b3 Mon Sep 17 00:00:00 2001 From: goodsoft Date: Tue, 26 Mar 2019 22:36:13 +0200 Subject: [PATCH 12/14] Add a migration to mark all invalid blocks as non-consensus (#1644) --- .../20190326202921_lose_consensus_for_invalid_blocks.sql | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 apps/explorer/priv/repo/migrations/scripts/20190326202921_lose_consensus_for_invalid_blocks.sql diff --git a/apps/explorer/priv/repo/migrations/scripts/20190326202921_lose_consensus_for_invalid_blocks.sql b/apps/explorer/priv/repo/migrations/scripts/20190326202921_lose_consensus_for_invalid_blocks.sql new file mode 100644 index 0000000000..38e8678484 --- /dev/null +++ b/apps/explorer/priv/repo/migrations/scripts/20190326202921_lose_consensus_for_invalid_blocks.sql @@ -0,0 +1,6 @@ +UPDATE blocks SET consensus = FALSE +WHERE number IN ( + SELECT b0.number - 1 FROM "blocks" AS b0 + LEFT JOIN "blocks" AS b1 ON (b0."parent_hash" = b1."hash") AND b1."consensus" + WHERE b0."number" > 0 AND b0."consensus" AND b1."hash" IS NULL +); From f5268fc4504e084b1f8110de37cc93b8fc2fae1d Mon Sep 17 00:00:00 2001 From: goodsoft Date: Wed, 27 Mar 2019 15:33:01 +0200 Subject: [PATCH 13/14] Address review comments --- .../explorer/chain/import/runner/blocks.ex | 20 ++++++++----------- ...2921_lose_consensus_for_invalid_blocks.sql | 4 ++-- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex index c351bfd218..100f120290 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex @@ -573,12 +573,10 @@ defmodule Explorer.Chain.Import.Runner.Blocks do initial = from(t in Transaction, where: false) Enum.reduce(blocks_changes, initial, fn %{consensus: consensus, hash: hash, number: number}, acc -> - case consensus do - false -> - from(transaction in acc, or_where: transaction.block_hash == ^hash and transaction.block_number == ^number) - - true -> - from(transaction in acc, or_where: transaction.block_hash != ^hash and transaction.block_number == ^number) + if consensus do + from(transaction in acc, or_where: transaction.block_hash != ^hash and transaction.block_number == ^number) + else + from(transaction in acc, or_where: transaction.block_hash == ^hash and transaction.block_number == ^number) end end) end @@ -587,12 +585,10 @@ defmodule Explorer.Chain.Import.Runner.Blocks do initial = from(b in Block, where: false) Enum.reduce(blocks_changes, initial, fn %{consensus: consensus, parent_hash: parent_hash, number: number}, acc -> - case consensus do - false -> - acc - - true -> - from(block in acc, or_where: block.number == ^(number - 1) and block.hash != ^parent_hash) + if consensus do + from(block in acc, or_where: block.number == ^(number - 1) and block.hash != ^parent_hash) + else + acc end end) end diff --git a/apps/explorer/priv/repo/migrations/scripts/20190326202921_lose_consensus_for_invalid_blocks.sql b/apps/explorer/priv/repo/migrations/scripts/20190326202921_lose_consensus_for_invalid_blocks.sql index 38e8678484..f201ccbece 100644 --- a/apps/explorer/priv/repo/migrations/scripts/20190326202921_lose_consensus_for_invalid_blocks.sql +++ b/apps/explorer/priv/repo/migrations/scripts/20190326202921_lose_consensus_for_invalid_blocks.sql @@ -1,5 +1,5 @@ -UPDATE blocks SET consensus = FALSE -WHERE number IN ( +UPDATE blocks SET consensus = FALSE, updated_at = NOW() +WHERE consensus AND number IN ( SELECT b0.number - 1 FROM "blocks" AS b0 LEFT JOIN "blocks" AS b1 ON (b0."parent_hash" = b1."hash") AND b1."consensus" WHERE b0."number" > 0 AND b0."consensus" AND b1."hash" IS NULL From 2bb87ffe9a27ad07d4558970ab17145cf99d136b Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Wed, 27 Mar 2019 18:13:06 +0300 Subject: [PATCH 14/14] Update changelog --- CHANGELOG.md | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2837e4210b..b04b7e809c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,9 +2,18 @@ ### Features - - [1611](https://github.com/poanetwork/blockscout/pull/1611) - allow setting the first indexing block - - [1596](https://github.com/poanetwork/blockscout/pull/1596) - add endpoint to create decompiled contracts - - [1634](https://github.com/poanetwork/blockscout/pull/1634) - add transaction count cache +### Fixes + +### Chore + + +## 1.3.8-beta + +### Features + + - [#1611](https://github.com/poanetwork/blockscout/pull/1611) - allow setting the first indexing block + - [#1596](https://github.com/poanetwork/blockscout/pull/1596) - add endpoint to create decompiled contracts + - [#1634](https://github.com/poanetwork/blockscout/pull/1634) - add transaction count cache ### Fixes