From dd2dbbfddbe31f34bd5d45d265d73f222e96350c Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Fri, 5 Apr 2019 11:09:02 +0300 Subject: [PATCH 01/13] allow setting update interval for addresses with balances counter --- apps/explorer/config/config.exs | 5 ++++- .../lib/explorer/counters/addresses_with_balance_counter.ex | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/apps/explorer/config/config.exs b/apps/explorer/config/config.exs index 2bf03ccd6b..b9ccad2739 100644 --- a/apps/explorer/config/config.exs +++ b/apps/explorer/config/config.exs @@ -13,7 +13,10 @@ config :explorer, config :explorer, Explorer.Counters.AverageBlockTime, enabled: true -config :explorer, Explorer.Counters.AddressesWithBalanceCounter, enabled: true, enable_consolidation: true +config :explorer, Explorer.Counters.AddressesWithBalanceCounter, + enabled: true, + enable_consolidation: true, + update_interval_in_seconds: System.get_env("ADDRESS_WITH_BALANCES_UPDATE_INTERVAL") || 30 * 60 config :explorer, Explorer.ExchangeRates, enabled: true, store: :ets diff --git a/apps/explorer/lib/explorer/counters/addresses_with_balance_counter.ex b/apps/explorer/lib/explorer/counters/addresses_with_balance_counter.ex index adbeb6e6f2..3a03a7bcd5 100644 --- a/apps/explorer/lib/explorer/counters/addresses_with_balance_counter.ex +++ b/apps/explorer/lib/explorer/counters/addresses_with_balance_counter.ex @@ -29,6 +29,8 @@ defmodule Explorer.Counters.AddressesWithBalanceCounter do config = Application.get_env(:explorer, Explorer.Counters.AddressesWithBalanceCounter) @enable_consolidation Keyword.get(config, :enable_consolidation) + @update_interval_in_seconds Keyword.get(config, :update_interval_in_seconds) + @doc """ Starts a process to periodically update the counter of the token holders. """ @@ -62,7 +64,7 @@ defmodule Explorer.Counters.AddressesWithBalanceCounter do defp schedule_next_consolidation do if enable_consolidation?() do - Process.send_after(self(), :consolidate, :timer.minutes(30)) + Process.send_after(self(), :consolidate, :timer.seconds(@update_interval_in_seconds)) end end From 714d6d82b2f27031280c62788371d4f4af04a859 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Fri, 5 Apr 2019 11:20:35 +0300 Subject: [PATCH 02/13] add CHANGELOG entry --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index da3e1fb897..106112de28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - [#1654](https://github.com/poanetwork/blockscout/pull/1654) - add decompiled code tab - [#1661](https://github.com/poanetwork/blockscout/pull/1661) - try to compile smart contract with the latest evm version - [#1665](https://github.com/poanetwork/blockscout/pull/1665) - Add contract verification RPC endpoint. + - [#1706](https://github.com/poanetwork/blockscout/pull/1706) - allow setting update interval for addresses with b ### Fixes @@ -16,7 +17,7 @@ - [#1684](https://github.com/poanetwork/blockscout/pull/1684) - Discard child block with parent_hash not matching hash of imported block - [#1699](https://github.com/poanetwork/blockscout/pull/1699) - use seconds as transaction cache period measure - [#1697](https://github.com/poanetwork/blockscout/pull/1697) - fix failing in rpc if balance is empty - + ### Chore From ad08a2b444a8c74e2bb13d4dc0e6e1c06b0547c0 Mon Sep 17 00:00:00 2001 From: Tim Mecklem Date: Sun, 7 Apr 2019 21:11:13 -0400 Subject: [PATCH 03/13] Update config.yml Fix a minor typo that could cause a dialyzer cache miss in circle ci --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0ca044b999..657681a305 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -189,7 +189,7 @@ jobs: - restore_cache: keys: - - v7-mix-dailyzer-{{ checksum "OTP_VERSION.lock" }}-{{ checksum "ELIXIR_VERSION.lock" }}-{{ checksum "mix.lock" }} + - v7-mix-dialyzer-{{ checksum "OTP_VERSION.lock" }}-{{ checksum "ELIXIR_VERSION.lock" }}-{{ checksum "mix.lock" }} - v7-mix-dialyzer-{{ checksum "OTP_VERSION.lock" }}-{{ checksum "ELIXIR_VERSION.lock" }}-{{ checksum "mix.exs" }} - v7-mix-dialyzer-{{ checksum "OTP_VERSION.lock" }}-{{ checksum "ELIXIR_VERSION.lock" }} From 08c4266d63d1059cb29638bc175906243a3c01b5 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Mon, 8 Apr 2019 12:59:32 +0300 Subject: [PATCH 04/13] rescue failing repo in block number cache update --- apps/explorer/lib/explorer/chain/block_number_cache.ex | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/apps/explorer/lib/explorer/chain/block_number_cache.ex b/apps/explorer/lib/explorer/chain/block_number_cache.ex index e4e6219ebf..28d8d83e9c 100644 --- a/apps/explorer/lib/explorer/chain/block_number_cache.ex +++ b/apps/explorer/lib/explorer/chain/block_number_cache.ex @@ -61,7 +61,7 @@ defmodule Explorer.Chain.BlockNumberCache do defp update_cache do current_time = current_time() - {min, max} = Chain.fetch_min_and_max_block_numbers() + {min, max} = min_and_max_from_db() tuple = {min, max, current_time} :ets.insert(@tab, {@key, tuple}) @@ -85,6 +85,13 @@ defmodule Explorer.Chain.BlockNumberCache do cache_period end + defp min_and_max_from_db do + Chain.fetch_min_and_max_block_numbers() + rescue + _e -> + {0, 0} + end + defp current_time do utc_now = DateTime.utc_now() From 992493a84e140d065a308e00dfcdd20b190b670a Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Mon, 8 Apr 2019 13:38:01 +0300 Subject: [PATCH 05/13] add CHANGELOG entry --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index da3e1fb897..3386303dcd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,8 @@ - [#1684](https://github.com/poanetwork/blockscout/pull/1684) - Discard child block with parent_hash not matching hash of imported block - [#1699](https://github.com/poanetwork/blockscout/pull/1699) - use seconds as transaction cache period measure - [#1697](https://github.com/poanetwork/blockscout/pull/1697) - fix failing in rpc if balance is empty - + - [#1711](https://github.com/poanetwork/blockscout/pull/1711) - rescue failing repo in block number cache update + ### Chore From 198ebcd140831f958846be012b489cbef6f150e1 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Mon, 8 Apr 2019 13:42:25 +0300 Subject: [PATCH 06/13] do not set contract code from transactio input --- apps/indexer/lib/indexer/address_extraction.ex | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/indexer/lib/indexer/address_extraction.ex b/apps/indexer/lib/indexer/address_extraction.ex index 0a4046a714..5fba0da10c 100644 --- a/apps/indexer/lib/indexer/address_extraction.ex +++ b/apps/indexer/lib/indexer/address_extraction.ex @@ -86,8 +86,7 @@ defmodule Indexer.AddressExtraction do transactions: [ [ %{from: :block_number, to: :fetched_coin_balance_block_number}, - %{from: :created_contract_address_hash, to: :hash}, - %{from: :input, to: :contract_code} + %{from: :created_contract_address_hash, to: :hash} ], [ %{from: :block_number, to: :fetched_coin_balance_block_number}, From df52a9defe2409e1a1ae541605c027373834697b Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Mon, 8 Apr 2019 15:46:47 +0300 Subject: [PATCH 07/13] fix average block time calculation --- .../lib/block_scout_web/notifier.ex | 2 +- .../explorer/counters/average_block_time.ex | 48 ++++++++----------- 2 files changed, 22 insertions(+), 28 deletions(-) diff --git a/apps/block_scout_web/lib/block_scout_web/notifier.ex b/apps/block_scout_web/lib/block_scout_web/notifier.ex index d1f538cf85..21a104848e 100644 --- a/apps/block_scout_web/lib/block_scout_web/notifier.ex +++ b/apps/block_scout_web/lib/block_scout_web/notifier.ex @@ -116,7 +116,7 @@ defmodule BlockScoutWeb.Notifier do defp broadcast_block(block) do preloaded_block = Repo.preload(block, [[miner: :names], :transactions, :rewards]) - average_block_time = AverageBlockTime.average_block_time(preloaded_block) + average_block_time = AverageBlockTime.average_block_time() Endpoint.broadcast("blocks:new_block", "new_block", %{ block: preloaded_block, diff --git a/apps/explorer/lib/explorer/counters/average_block_time.ex b/apps/explorer/lib/explorer/counters/average_block_time.ex index 11e26e266b..a67e85a19e 100644 --- a/apps/explorer/lib/explorer/counters/average_block_time.ex +++ b/apps/explorer/lib/explorer/counters/average_block_time.ex @@ -11,6 +11,8 @@ defmodule Explorer.Counters.AverageBlockTime do alias Explorer.Repo alias Timex.Duration + @refresh_period 30 * 60 * 1_000 + @doc """ Starts a process to periodically update the counter of the token holders. """ @@ -19,15 +21,14 @@ defmodule Explorer.Counters.AverageBlockTime do GenServer.start_link(__MODULE__, :ok, name: __MODULE__) end - def average_block_time(block \\ nil) do + def average_block_time do enabled? = :explorer |> Application.fetch_env!(__MODULE__) |> Keyword.fetch!(:enabled) if enabled? do - block = if block, do: {block.number, DateTime.to_unix(block.timestamp, :millisecond)} - GenServer.call(__MODULE__, {:average_block_time, block}) + GenServer.call(__MODULE__, :average_block_time) else {:error, :disabled} end @@ -36,6 +37,22 @@ defmodule Explorer.Counters.AverageBlockTime do ## Server @impl true def init(_) do + Process.send_after(self(), :refresh_timestamps, @refresh_period) + + {:ok, refresh_timestamps()} + end + + @impl true + def handle_call(:average_block_time, _from, %{average: average} = state), do: {:reply, average, state} + + @impl true + def handle_info(:refresh_timestamps, _) do + Process.send_after(self(), :refresh_timestamps, @refresh_period) + + {:ok, refresh_timestamps()} + end + + defp refresh_timestamps do timestamps_query = from(block in Block, limit: 100, @@ -51,30 +68,7 @@ defmodule Explorer.Counters.AverageBlockTime do {number, DateTime.to_unix(timestamp, :millisecond)} end) - {:ok, %{timestamps: timestamps, average: average_distance(timestamps)}} - end - - @impl true - def handle_call({:average_block_time, nil}, _from, %{average: average} = state), do: {:reply, average, state} - - def handle_call({:average_block_time, block}, _from, state) do - state = add_block(state, block) - {:reply, state.average, state} - end - - # This is pretty naive, but we'll only ever be sorting 100 dates so I don't think - # complex logic is really necessary here. - defp add_block(%{timestamps: timestamps} = state, {new_number, _} = block) do - if Enum.any?(timestamps, fn {number, _} -> number == new_number end) do - state - else - timestamps = - [block | timestamps] - |> Enum.sort_by(fn {number, _} -> number end, &Kernel.>/2) - |> Enum.take(100) - - %{state | timestamps: timestamps, average: average_distance(timestamps)} - end + %{timestamps: timestamps, average: average_distance(timestamps)} end defp average_distance([]), do: Duration.from_milliseconds(0) From 6d21e8529eb09f8761811b81b1dc530091917012 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Mon, 8 Apr 2019 15:59:09 +0300 Subject: [PATCH 08/13] fix converting to integer --- apps/explorer/config/config.exs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/apps/explorer/config/config.exs b/apps/explorer/config/config.exs index b9ccad2739..eda40bc548 100644 --- a/apps/explorer/config/config.exs +++ b/apps/explorer/config/config.exs @@ -13,10 +13,18 @@ config :explorer, config :explorer, Explorer.Counters.AverageBlockTime, enabled: true +balances_update_interval = + if System.get_env("ADDRESS_WITH_BALANCES_UPDATE_INTERVAL") do + case Integer.parse(System.get_env("ADDRESS_WITH_BALANCES_UPDATE_INTERVAL")) do + {integer, ""} -> integer * 1_000 + _ -> nil + end + end + config :explorer, Explorer.Counters.AddressesWithBalanceCounter, enabled: true, enable_consolidation: true, - update_interval_in_seconds: System.get_env("ADDRESS_WITH_BALANCES_UPDATE_INTERVAL") || 30 * 60 + update_interval_in_seconds: balances_update_interval || 30 * 60 config :explorer, Explorer.ExchangeRates, enabled: true, store: :ets From 02f3d922fe11aa6195c99e25e0ec1bf7b72cf818 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Mon, 8 Apr 2019 16:00:24 +0300 Subject: [PATCH 09/13] use seconds --- apps/explorer/config/config.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/explorer/config/config.exs b/apps/explorer/config/config.exs index eda40bc548..dea1be2b46 100644 --- a/apps/explorer/config/config.exs +++ b/apps/explorer/config/config.exs @@ -16,7 +16,7 @@ config :explorer, Explorer.Counters.AverageBlockTime, enabled: true balances_update_interval = if System.get_env("ADDRESS_WITH_BALANCES_UPDATE_INTERVAL") do case Integer.parse(System.get_env("ADDRESS_WITH_BALANCES_UPDATE_INTERVAL")) do - {integer, ""} -> integer * 1_000 + {integer, ""} -> integer _ -> nil end end From ae667dfe4314a13184759bab808f34d89ea04d74 Mon Sep 17 00:00:00 2001 From: Paul Tsupikoff Date: Tue, 9 Apr 2019 00:17:05 +0300 Subject: [PATCH 10/13] Add a checklist to the PR template (#1693) We tend to forget adding changelog entries for PRs, so here's an extra reminder --- CHANGELOG.md | 2 ++ PULL_REQUEST_TEMPLATE.md | 17 +++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index da3e1fb897..8c57779a55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,8 @@ ### Chore + - [#1693](https://github.com/poanetwork/blockscout/pull/1693) - Add a checklist to the PR template + ## 1.3.8-beta diff --git a/PULL_REQUEST_TEMPLATE.md b/PULL_REQUEST_TEMPLATE.md index 03444118b9..3ac2a5a1d1 100644 --- a/PULL_REQUEST_TEMPLATE.md +++ b/PULL_REQUEST_TEMPLATE.md @@ -18,3 +18,20 @@ ## Upgrading *If you have any Incompatible Changes in the above Changelog, outline how users of prior versions can upgrade once this PR lands or when reviewers are testing locally. A common upgrading step is "Database reset and re-index required".* + +## Checklist for your PR + + + + - [ ] I added an entry to `CHANGELOG.md` with this PR + - [ ] If I added new functionality, I added tests covering it. + - [ ] If I fixed a bug, I added a regression test to prevent the bug from silently reappearing again. + - [ ] I checked whether I should update the docs and did so if necessary From 6360ceb4de2665552567a9cf72a97557aa2e4404 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 9 Apr 2019 12:13:09 +0300 Subject: [PATCH 11/13] fix dialyzer --- apps/explorer/lib/explorer/counters/average_block_time.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/explorer/lib/explorer/counters/average_block_time.ex b/apps/explorer/lib/explorer/counters/average_block_time.ex index a67e85a19e..f5d22374cf 100644 --- a/apps/explorer/lib/explorer/counters/average_block_time.ex +++ b/apps/explorer/lib/explorer/counters/average_block_time.ex @@ -49,7 +49,7 @@ defmodule Explorer.Counters.AverageBlockTime do def handle_info(:refresh_timestamps, _) do Process.send_after(self(), :refresh_timestamps, @refresh_period) - {:ok, refresh_timestamps()} + {:noreply, refresh_timestamps()} end defp refresh_timestamps do From e0ac68f22e4480d7e643ba10b3272e59c518d04c Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 9 Apr 2019 12:17:57 +0300 Subject: [PATCH 12/13] add CHAGELOG entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4403c81c37..d5a8aa60b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ - [#1699](https://github.com/poanetwork/blockscout/pull/1699) - use seconds as transaction cache period measure - [#1697](https://github.com/poanetwork/blockscout/pull/1697) - fix failing in rpc if balance is empty - [#1711](https://github.com/poanetwork/blockscout/pull/1711) - rescue failing repo in block number cache update + - [#1714](https://github.com/poanetwork/blockscout/pull/1714) - fix average block time calculation ### Chore From 4444f15e62d2955371f9ac9941cd6527966dfa18 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 9 Apr 2019 13:52:18 +0300 Subject: [PATCH 13/13] fix tests --- .../explorer/counters/average_block_time.ex | 11 +++++- .../counters/average_block_time_test.exs | 38 ------------------- .../coin_balance/on_demand_fetcher_test.exs | 14 +++---- 3 files changed, 16 insertions(+), 47 deletions(-) diff --git a/apps/explorer/lib/explorer/counters/average_block_time.ex b/apps/explorer/lib/explorer/counters/average_block_time.ex index f5d22374cf..27a524c22a 100644 --- a/apps/explorer/lib/explorer/counters/average_block_time.ex +++ b/apps/explorer/lib/explorer/counters/average_block_time.ex @@ -34,6 +34,10 @@ defmodule Explorer.Counters.AverageBlockTime do end end + def refresh do + GenServer.call(__MODULE__, :refresh_timestamps) + end + ## Server @impl true def init(_) do @@ -45,6 +49,11 @@ defmodule Explorer.Counters.AverageBlockTime do @impl true def handle_call(:average_block_time, _from, %{average: average} = state), do: {:reply, average, state} + @impl true + def handle_call(:refresh_timestamps, _, _) do + {:reply, :ok, refresh_timestamps()} + end + @impl true def handle_info(:refresh_timestamps, _) do Process.send_after(self(), :refresh_timestamps, @refresh_period) @@ -56,7 +65,7 @@ defmodule Explorer.Counters.AverageBlockTime do timestamps_query = from(block in Block, limit: 100, - offset: 1, + offset: 0, order_by: [desc: block.number], select: {block.number, block.timestamp} ) diff --git a/apps/explorer/test/explorer/counters/average_block_time_test.exs b/apps/explorer/test/explorer/counters/average_block_time_test.exs index 27ef74d551..63c47042c7 100644 --- a/apps/explorer/test/explorer/counters/average_block_time_test.exs +++ b/apps/explorer/test/explorer/counters/average_block_time_test.exs @@ -5,8 +5,6 @@ defmodule Explorer.Counters.AverageBlockTimeTest do alias Explorer.Counters.AverageBlockTime - defp block(number, last, duration), do: %{number: number, timestamp: Timex.shift(last, seconds: duration)} - setup do start_supervised!(AverageBlockTime) Application.put_env(:explorer, AverageBlockTime, enabled: true) @@ -26,41 +24,5 @@ defmodule Explorer.Counters.AverageBlockTimeTest do test "without blocks duration is 0" do assert AverageBlockTime.average_block_time() == Timex.Duration.parse!("PT0S") end - - test "with only one block, the duration is 0" do - now = Timex.now() - block = block(0, now, 0) - - assert AverageBlockTime.average_block_time(block) == Timex.Duration.parse!("PT0S") - end - - test "once there are two blocks, the duration is the average distance between them all" do - now = Timex.now() - - block0 = block(0, now, 0) - block1 = block(1, now, 2) - block2 = block(2, now, 6) - - AverageBlockTime.average_block_time(block0) - assert AverageBlockTime.average_block_time(block1) == Timex.Duration.parse!("PT2S") - assert AverageBlockTime.average_block_time(block2) == Timex.Duration.parse!("PT3S") - end - - test "only the last 100 blocks are considered" do - now = Timex.now() - - block0 = block(0, now, 0) - block1 = block(1, now, 2000) - - AverageBlockTime.average_block_time(block0) - AverageBlockTime.average_block_time(block1) - - for i <- 1..100 do - block = block(i + 1, now, 2000 + i) - AverageBlockTime.average_block_time(block) - end - - assert AverageBlockTime.average_block_time() == Timex.Duration.parse!("PT1S") - end end end diff --git a/apps/indexer/test/indexer/coin_balance/on_demand_fetcher_test.exs b/apps/indexer/test/indexer/coin_balance/on_demand_fetcher_test.exs index 071493e28a..590ac6b7da 100644 --- a/apps/indexer/test/indexer/coin_balance/on_demand_fetcher_test.exs +++ b/apps/indexer/test/indexer/coin_balance/on_demand_fetcher_test.exs @@ -41,10 +41,9 @@ defmodule Indexer.CoinBalance.OnDemandFetcherTest do # we space these very far apart so that we know it will consider the 0th block stale (it calculates how far # back we'd need to go to get 24 hours in the past) - block_0 = insert(:block, number: 0, timestamp: Timex.shift(now, hours: -50)) - AverageBlockTime.average_block_time(block_0) - block_1 = insert(:block, number: 1, timestamp: now) - AverageBlockTime.average_block_time(block_1) + insert(:block, number: 0, timestamp: Timex.shift(now, hours: -50)) + insert(:block, number: 1, timestamp: now) + AverageBlockTime.refresh() stale_address = insert(:address, fetched_coin_balance: 1, fetched_coin_balance_block_number: 0) current_address = insert(:address, fetched_coin_balance: 1, fetched_coin_balance_block_number: 1) @@ -89,10 +88,9 @@ defmodule Indexer.CoinBalance.OnDemandFetcherTest do # we space these very far apart so that we know it will consider the 0th block stale (it calculates how far # back we'd need to go to get 24 hours in the past) - block_0 = insert(:block, number: 0, timestamp: Timex.shift(now, hours: -50)) - AverageBlockTime.average_block_time(block_0) - block_1 = insert(:block, number: 1, timestamp: now) - AverageBlockTime.average_block_time(block_1) + insert(:block, number: 0, timestamp: Timex.shift(now, hours: -50)) + insert(:block, number: 1, timestamp: now) + AverageBlockTime.refresh() :ok end