From 978af2df39c1833de288befcc2220de8ff76e9f3 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Fri, 22 Mar 2019 12:11:49 +0300 Subject: [PATCH 1/8] add transaction count cache --- .../explorer/chain/transaction_count_cache.ex | 115 ++++++++++++++++++ .../chain/transaction_count_cache_test.exs | 58 +++++++++ 2 files changed, 173 insertions(+) create mode 100644 apps/explorer/lib/explorer/chain/transaction_count_cache.ex create mode 100644 apps/explorer/test/explorer/chain/transaction_count_cache_test.exs diff --git a/apps/explorer/lib/explorer/chain/transaction_count_cache.ex b/apps/explorer/lib/explorer/chain/transaction_count_cache.ex new file mode 100644 index 0000000000..9dc6be2e68 --- /dev/null +++ b/apps/explorer/lib/explorer/chain/transaction_count_cache.ex @@ -0,0 +1,115 @@ +defmodule Explorer.Chain.TransactionCountCache do + @moduledoc """ + Cache for estimated transaction count. + """ + + use GenServer + + alias Explorer.Chain.Transaction + alias Explorer.Repo + + @tab :transaction_count_cache + # 2 hours + @cache_period 1_000 * 60 * 60 * 2 + @default_value 0 + @key "count" + @name __MODULE__ + + def start_link([params, gen_server_options]) do + GenServer.start_link(__MODULE__, params, name: gen_server_options[:name] || @name) + end + + def init(params) do + cache_period = params[:cache_period] || @cache_period + current_value = params[:default_value] || @default_value + + init_ets_table() + + schedule_cache_update() + + {:ok, {{cache_period, current_value}, nil}} + end + + def value(process_name \\ __MODULE__) do + GenServer.call(process_name, :value) + end + + def handle_call(:value, _, {{cache_period, default_value}, task}) do + {value, task} = + case cached_values() do + nil -> + {default_value, update_cache(task)} + + {cached_value, timestamp} -> + task = + if current_time() - timestamp > cache_period do + update_cache(task) + end + + {cached_value, task} + end + + {:reply, value, {{cache_period, default_value}, task}} + end + + def update_cache(nil) do + async_update_cache() + end + + def update_cache(task) do + task + end + + def handle_cast({:update_cache, value}, {{cache_period, default_value}, _}) do + current_time = current_time() + tuple = {value, current_time} + + :ets.insert(@tab, {@key, tuple}) + + {:noreply, {{cache_period, default_value}, nil}} + end + + def handle_info({:DOWN, _, _, _, _}, {{cache_period, default_value}, _}) do + {:noreply, {{cache_period, default_value}, nil}} + end + + def handle_info(_, {{cache_period, default_value}, _}) do + {:noreply, {{cache_period, default_value}, nil}} + end + + def async_update_cache do + Task.async(fn -> + result = Repo.aggregate(Transaction, :count, :hash, timeout: :infinity) + + GenServer.cast(__MODULE__, {:update_cache, result}) + end) + end + + defp init_ets_table do + if :ets.whereis(@tab) == :undefined do + :ets.new(@tab, [ + :set, + :named_table, + :public, + write_concurrency: true + ]) + end + end + + defp cached_values do + case :ets.lookup(@tab, @key) do + [{_, cached_values}] -> cached_values + _ -> nil + end + end + + defp schedule_cache_update do + Process.send_after(self(), :update_cache, 2_000) + end + + defp current_time do + utc_now = DateTime.utc_now() + + DateTime.to_unix(utc_now, :millisecond) + end +end diff --git a/apps/explorer/test/explorer/chain/transaction_count_cache_test.exs b/apps/explorer/test/explorer/chain/transaction_count_cache_test.exs new file mode 100644 index 0000000000..50c1e818a2 --- /dev/null +++ b/apps/explorer/test/explorer/chain/transaction_count_cache_test.exs @@ -0,0 +1,58 @@ +defmodule Explorer.Chain.TransactionCountCacheTest do + use Explorer.DataCase + + alias Explorer.Chain.TransactionCountCache + + test "returns default transaction count" do + TransactionCountCache.start_link([[], []]) + + result = TransactionCountCache.value() + + assert result == 0 + end + + test "updates cache if initial value is zero" do + TransactionCountCache.start_link([[], []]) + + insert(:transaction) + insert(:transaction) + + result = TransactionCountCache.value() + + assert result == 0 + + Process.sleep(500) + + updated_value = TransactionCountCache.value() + + assert updated_value == 2 + end + + test "does not update cache if cache period did not pass" do + TransactionCountCache.start_link([[], []]) + + insert(:transaction) + insert(:transaction) + + result = TransactionCountCache.value() + + assert result == 0 + + Process.sleep(500) + + updated_value = TransactionCountCache.value() + + assert updated_value == 2 + + insert(:transaction) + insert(:transaction) + + _updated_value = TransactionCountCache.value() + + Process.sleep(500) + + updated_value = TransactionCountCache.value() + + assert updated_value == 2 + end +end From 8a58d4b03eb138013293b952b1329661752e7d3f Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Fri, 22 Mar 2019 13:40:30 +0300 Subject: [PATCH 2/8] use cache for estimated transaction count --- apps/explorer/lib/explorer/application.ex | 4 +++- apps/explorer/lib/explorer/chain.ex | 7 ++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/apps/explorer/lib/explorer/application.ex b/apps/explorer/lib/explorer/application.ex index ecf4250213..b896e84a1f 100644 --- a/apps/explorer/lib/explorer/application.ex +++ b/apps/explorer/lib/explorer/application.ex @@ -8,6 +8,7 @@ defmodule Explorer.Application do alias Explorer.Admin alias Explorer.Chain.BlockNumberCache alias Explorer.Repo.PrometheusLogger + alias Explorer.Chain.TransactionCountCache @impl Application def start(_type, _args) do @@ -27,7 +28,8 @@ defmodule Explorer.Application do Supervisor.child_spec({Task.Supervisor, name: Explorer.MarketTaskSupervisor}, id: Explorer.MarketTaskSupervisor), Supervisor.child_spec({Task.Supervisor, name: Explorer.TaskSupervisor}, id: Explorer.TaskSupervisor), {Registry, keys: :duplicate, name: Registry.ChainEvents, id: Registry.ChainEvents}, - {Admin.Recovery, [[], [name: Admin.Recovery]]} + {Admin.Recovery, [[], [name: Admin.Recovery]]}, + {TransactionCountCache, [[], []]} ] children = base_children ++ configurable_children() diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index 58b6e1a34b..5bd6394606 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -20,7 +20,6 @@ defmodule Explorer.Chain do import EthereumJSONRPC, only: [integer_to_quantity: 1] - alias Ecto.Adapters.SQL alias Ecto.{Changeset, Multi} alias Explorer.Chain.{ @@ -39,6 +38,7 @@ defmodule Explorer.Chain do Token, TokenTransfer, Transaction, + TransactionCountCache, Wei } @@ -1842,10 +1842,7 @@ defmodule Explorer.Chain do """ @spec transaction_estimated_count() :: non_neg_integer() def transaction_estimated_count do - %Postgrex.Result{rows: [[rows]]} = - SQL.query!(Repo, "SELECT reltuples::BIGINT AS estimate FROM pg_class WHERE relname='transactions'") - - rows + TransactionCountCache.value() end @doc """ From 6360ecf7997d61023ee3eeaa43b28d88ed4a8768 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Fri, 22 Mar 2019 14:56:34 +0300 Subject: [PATCH 3/8] fix tests --- .../explorer/chain/transaction_count_cache.ex | 78 ++++++++++++------- .../chain/transaction_count_cache_test.exs | 30 ++++--- 2 files changed, 63 insertions(+), 45 deletions(-) diff --git a/apps/explorer/lib/explorer/chain/transaction_count_cache.ex b/apps/explorer/lib/explorer/chain/transaction_count_cache.ex index 9dc6be2e68..463a6d6cec 100644 --- a/apps/explorer/lib/explorer/chain/transaction_count_cache.ex +++ b/apps/explorer/lib/explorer/chain/transaction_count_cache.ex @@ -3,12 +3,13 @@ defmodule Explorer.Chain.TransactionCountCache do Cache for estimated transaction count. """ + require Logger + use GenServer alias Explorer.Chain.Transaction alias Explorer.Repo - @tab :transaction_count_cache # 2 hours @cache_period 1_000 * 60 * 60 * 2 @default_value 0 @@ -16,78 +17,97 @@ defmodule Explorer.Chain.TransactionCountCache do @name __MODULE__ def start_link([params, gen_server_options]) do - GenServer.start_link(__MODULE__, params, name: gen_server_options[:name] || @name) + name = gen_server_options[:name] || @name + params_with_name = Keyword.put(params, :name, name) + + GenServer.start_link(__MODULE__, params_with_name, name: name) end def init(params) do cache_period = params[:cache_period] || @cache_period current_value = params[:default_value] || @default_value + name = params[:name] - init_ets_table() + init_ets_table(name) schedule_cache_update() - {:ok, {{cache_period, current_value}, nil}} + {:ok, {{cache_period, current_value, name}, nil}} end def value(process_name \\ __MODULE__) do GenServer.call(process_name, :value) end - def handle_call(:value, _, {{cache_period, default_value}, task}) do + def handle_call(:value, _, {{cache_period, default_value, name}, task}) do {value, task} = - case cached_values() do + case cached_values(name) do nil -> - {default_value, update_cache(task)} + {default_value, update_cache(task, name)} {cached_value, timestamp} -> task = if current_time() - timestamp > cache_period do - update_cache(task) + update_cache(task, name) end {cached_value, task} end - {:reply, value, {{cache_period, default_value}, task}} + {:reply, value, {{cache_period, default_value, name}, task}} end - def update_cache(nil) do - async_update_cache() + def update_cache(nil, name) do + async_update_cache(name) end - def update_cache(task) do + def update_cache(task, _) do task end - def handle_cast({:update_cache, value}, {{cache_period, default_value}, _}) do + def handle_cast({:update_cache, value}, {{cache_period, default_value, name}, _}) do current_time = current_time() tuple = {value, current_time} - :ets.insert(@tab, {@key, tuple}) + table_name = table_name(name) + + :ets.insert(table_name, {@key, tuple}) - {:noreply, {{cache_period, default_value}, nil}} + {:noreply, {{cache_period, default_value, name}, nil}} end - def handle_info({:DOWN, _, _, _, _}, {{cache_period, default_value}, _}) do - {:noreply, {{cache_period, default_value}, nil}} + def handle_info({:DOWN, _, _, _, _}, {{cache_period, default_value, name}, _}) do + {:noreply, {{cache_period, default_value, name}, nil}} end - def handle_info(_, {{cache_period, default_value}, _}) do - {:noreply, {{cache_period, default_value}, nil}} + def handle_info(_, {{cache_period, default_value, name}, _}) do + {:noreply, {{cache_period, default_value, name}, nil}} end - def async_update_cache do - Task.async(fn -> - result = Repo.aggregate(Transaction, :count, :hash, timeout: :infinity) + defp table_name(name) do + name |> Atom.to_string() |> Macro.underscore() |> String.to_atom() + end - GenServer.cast(__MODULE__, {:update_cache, result}) + def async_update_cache(name) do + Task.async(fn -> + try do + result = Repo.aggregate(Transaction, :count, :hash, timeout: :infinity) + + GenServer.cast(name, {:update_cache, result}) + rescue + e -> + Logger.debug([ + "Coudn't update transaction count test #{inspect(e)}" + ]) + end end) end - defp init_ets_table do - if :ets.whereis(@tab) == :undefined do - :ets.new(@tab, [ + defp init_ets_table(name) do + table_name = table_name(name) + + if :ets.whereis(table_name) == :undefined do + :ets.new(table_name, [ :set, :named_table, :public, @@ -96,8 +116,10 @@ defmodule Explorer.Chain.TransactionCountCache do end end - defp cached_values do - case :ets.lookup(@tab, @key) do + defp cached_values(name) do + table_name = table_name(name) + + case :ets.lookup(table_name, @key) do [{_, cached_values}] -> cached_values _ -> nil end diff --git a/apps/explorer/test/explorer/chain/transaction_count_cache_test.exs b/apps/explorer/test/explorer/chain/transaction_count_cache_test.exs index 50c1e818a2..8ddcda2f04 100644 --- a/apps/explorer/test/explorer/chain/transaction_count_cache_test.exs +++ b/apps/explorer/test/explorer/chain/transaction_count_cache_test.exs @@ -4,54 +4,50 @@ defmodule Explorer.Chain.TransactionCountCacheTest do alias Explorer.Chain.TransactionCountCache test "returns default transaction count" do - TransactionCountCache.start_link([[], []]) + TransactionCountCache.start_link([[], [name: TestCache]]) - result = TransactionCountCache.value() + result = TransactionCountCache.value(TestCache) assert result == 0 end test "updates cache if initial value is zero" do - TransactionCountCache.start_link([[], []]) + TransactionCountCache.start_link([[], [name: TestCache]]) insert(:transaction) insert(:transaction) - result = TransactionCountCache.value() + _result = TransactionCountCache.value(TestCache) - assert result == 0 - - Process.sleep(500) + Process.sleep(1000) - updated_value = TransactionCountCache.value() + updated_value = TransactionCountCache.value(TestCache) assert updated_value == 2 end test "does not update cache if cache period did not pass" do - TransactionCountCache.start_link([[], []]) + TransactionCountCache.start_link([[], [name: TestCache]]) insert(:transaction) insert(:transaction) - result = TransactionCountCache.value() - - assert result == 0 + _result = TransactionCountCache.value(TestCache) - Process.sleep(500) + Process.sleep(1000) - updated_value = TransactionCountCache.value() + updated_value = TransactionCountCache.value(TestCache) assert updated_value == 2 insert(:transaction) insert(:transaction) - _updated_value = TransactionCountCache.value() + _updated_value = TransactionCountCache.value(TestCache) - Process.sleep(500) + Process.sleep(1000) - updated_value = TransactionCountCache.value() + updated_value = TransactionCountCache.value(TestCache) assert updated_value == 2 end From 6727a8bd5939c44f1d99e34f95325fcd1a8d0b03 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Mon, 25 Mar 2019 13:56:29 +0300 Subject: [PATCH 4/8] fix credo --- apps/explorer/lib/explorer/application.ex | 3 +-- apps/explorer/lib/explorer/chain/transaction_count_cache.ex | 6 +++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/apps/explorer/lib/explorer/application.ex b/apps/explorer/lib/explorer/application.ex index b896e84a1f..65f91c1d4b 100644 --- a/apps/explorer/lib/explorer/application.ex +++ b/apps/explorer/lib/explorer/application.ex @@ -6,9 +6,8 @@ defmodule Explorer.Application do use Application alias Explorer.Admin - alias Explorer.Chain.BlockNumberCache + alias Explorer.Chain.{BlockNumberCache, TransactionCountCache} alias Explorer.Repo.PrometheusLogger - alias Explorer.Chain.TransactionCountCache @impl Application def start(_type, _args) do diff --git a/apps/explorer/lib/explorer/chain/transaction_count_cache.ex b/apps/explorer/lib/explorer/chain/transaction_count_cache.ex index 463a6d6cec..809453338a 100644 --- a/apps/explorer/lib/explorer/chain/transaction_count_cache.ex +++ b/apps/explorer/lib/explorer/chain/transaction_count_cache.ex @@ -84,8 +84,12 @@ defmodule Explorer.Chain.TransactionCountCache do {:noreply, {{cache_period, default_value, name}, nil}} end + # sobelow_skip ["DOS"] defp table_name(name) do - name |> Atom.to_string() |> Macro.underscore() |> String.to_atom() + name + |> Atom.to_string() + |> Macro.underscore() + |> String.to_atom() end def async_update_cache(name) do From fd08bc1b3a188684469d58068c06ea728dfb3ada Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Mon, 25 Mar 2019 16:49:17 +0300 Subject: [PATCH 5/8] add CHANGELOG entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 684ed983e1..2f9e9e3a4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - [1611](https://github.com/poanetwork/blockscout/pull/1611) - allow setting the first indexing block - [1596](https://github.com/poanetwork/blockscout/pull/1596) - add endpoint to create decompiled contracts + - [1634](https://github.com/poanetwork/blockscout/pull/1634) - add transaction count cache ### Fixes From a6c4dce440be8d8e82d45e245f749d41160a26a7 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 26 Mar 2019 11:13:52 +0300 Subject: [PATCH 6/8] fetch transaction period from env variables --- apps/explorer/lib/explorer/chain/transaction_count_cache.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/explorer/lib/explorer/chain/transaction_count_cache.ex b/apps/explorer/lib/explorer/chain/transaction_count_cache.ex index 809453338a..884e1c7d1f 100644 --- a/apps/explorer/lib/explorer/chain/transaction_count_cache.ex +++ b/apps/explorer/lib/explorer/chain/transaction_count_cache.ex @@ -24,7 +24,7 @@ defmodule Explorer.Chain.TransactionCountCache do end def init(params) do - cache_period = params[:cache_period] || @cache_period + cache_period = System.get_env("TRANSACTION_COUNT_CACHE_PERIOD") || params[:cache_period] || @cache_period current_value = params[:default_value] || @default_value name = params[:name] From 62990e87cd49787ab6395994e73772af403ec3e0 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 26 Mar 2019 12:50:39 +0300 Subject: [PATCH 7/8] set only hours in env var --- .../explorer/lib/explorer/chain/transaction_count_cache.ex | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/apps/explorer/lib/explorer/chain/transaction_count_cache.ex b/apps/explorer/lib/explorer/chain/transaction_count_cache.ex index 884e1c7d1f..fa88cc5e9b 100644 --- a/apps/explorer/lib/explorer/chain/transaction_count_cache.ex +++ b/apps/explorer/lib/explorer/chain/transaction_count_cache.ex @@ -24,7 +24,7 @@ defmodule Explorer.Chain.TransactionCountCache do end def init(params) do - cache_period = System.get_env("TRANSACTION_COUNT_CACHE_PERIOD") || params[:cache_period] || @cache_period + cache_period = period_from_env_var() || params[:cache_period] || @cache_period current_value = params[:default_value] || @default_value name = params[:name] @@ -138,4 +138,9 @@ defmodule Explorer.Chain.TransactionCountCache do DateTime.to_unix(utc_now, :millisecond) end + + defp period_from_env_var do + env_var = System.get_env("TXS_COUNT_CACHE_PERIOD") + env_var && env_var * 1_000 * 60 * 60 + end end From dd9a0837bb28bc3dd2798399d7feb6fd08e371ff Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 26 Mar 2019 13:06:58 +0300 Subject: [PATCH 8/8] fix dialyzer --- .../lib/explorer/chain/transaction_count_cache.ex | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/apps/explorer/lib/explorer/chain/transaction_count_cache.ex b/apps/explorer/lib/explorer/chain/transaction_count_cache.ex index fa88cc5e9b..60ecc05285 100644 --- a/apps/explorer/lib/explorer/chain/transaction_count_cache.ex +++ b/apps/explorer/lib/explorer/chain/transaction_count_cache.ex @@ -140,7 +140,15 @@ defmodule Explorer.Chain.TransactionCountCache do end defp period_from_env_var do - env_var = System.get_env("TXS_COUNT_CACHE_PERIOD") - env_var && env_var * 1_000 * 60 * 60 + case System.get_env("TXS_COUNT_CACHE_PERIOD") do + value when is_binary(value) -> + case Integer.parse(value) do + {integer, ""} -> integer * 1_000 * 60 * 60 + _ -> nil + end + + _ -> + nil + end end end