From 32663bc0279c4a5f4eab6568cfc482cbd391ac2b Mon Sep 17 00:00:00 2001 From: Viktor Baranov Date: Mon, 7 Jun 2021 14:10:25 +0300 Subject: [PATCH] Speedup intial query for catchup fetcher --- .github/workflows/config.yml | 24 ++++----- CHANGELOG.md | 1 + apps/explorer/lib/explorer/application.ex | 4 +- apps/explorer/lib/explorer/chain.ex | 35 ++++++++++++- .../lib/explorer/chain/cache/block_number.ex | 2 - .../explorer/chain/cache/min_missing_block.ex | 52 +++++++++++++++++++ ...0210524165427_min_missing_block_number.exs | 12 +++++ apps/indexer/config/config.exs | 2 +- .../lib/indexer/block/catchup/fetcher.ex | 12 ++++- 9 files changed, 125 insertions(+), 19 deletions(-) create mode 100644 apps/explorer/lib/explorer/chain/cache/min_missing_block.ex create mode 100644 apps/explorer/priv/repo/migrations/20210524165427_min_missing_block_number.exs diff --git a/.github/workflows/config.yml b/.github/workflows/config.yml index 722f60c84c..65e9dfa7bf 100644 --- a/.github/workflows/config.yml +++ b/.github/workflows/config.yml @@ -19,7 +19,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-elixir@v1 with: - otp-version: '23.2.7' + otp-version: '23.3.4.1' elixir-version: '1.11.3' - run: mix local.hex --force - run: mix local.rebar --force @@ -46,7 +46,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-elixir@v1 with: - otp-version: '23.2.7' + otp-version: '23.3.4.1' elixir-version: '1.11.3' - run: mix local.hex --force - run: mix local.rebar --force @@ -61,7 +61,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-elixir@v1 with: - otp-version: '23.2.7' + otp-version: '23.3.4.1' elixir-version: '1.11.3' - run: mix local.hex --force - run: mix local.rebar --force @@ -76,7 +76,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-elixir@v1 with: - otp-version: '23.2.7' + otp-version: '23.3.4.1' elixir-version: '1.11.3' - run: mix local.hex --force - run: mix local.rebar --force @@ -103,7 +103,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-elixir@v1 with: - otp-version: '23.2.7' + otp-version: '23.3.4.1' elixir-version: '1.11.3' - run: mix local.hex --force - run: mix local.rebar --force @@ -121,7 +121,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-elixir@v1 with: - otp-version: '23.2.7' + otp-version: '23.3.4.1' elixir-version: '1.11.3' - run: mix local.hex --force - run: mix local.rebar --force @@ -142,7 +142,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-elixir@v1 with: - otp-version: '23.2.7' + otp-version: '23.3.4.1' elixir-version: '1.11.3' - run: mix local.hex --force - run: mix local.rebar --force @@ -167,7 +167,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-elixir@v1 with: - otp-version: '23.2.7' + otp-version: '23.3.4.1' elixir-version: '1.11.3' - run: mix local.hex --force - run: mix local.rebar --force @@ -211,7 +211,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-elixir@v1 with: - otp-version: '23.2.7' + otp-version: '23.3.4.1' elixir-version: '1.11.3' - run: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y - run: echo 'export PATH=~/.cargo/bin/:$PATH' >> $GITHUB_ENV @@ -267,7 +267,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-elixir@v1 with: - otp-version: '23.2.7' + otp-version: '23.3.4.1' elixir-version: '1.11.3' - run: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y - run: echo 'export PATH=~/.cargo/bin/:$PATH' >> $GITHUB_ENV @@ -325,7 +325,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-elixir@v1 with: - otp-version: '23.2.7' + otp-version: '23.3.4.1' elixir-version: '1.11.3' - run: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y - run: echo 'export PATH=~/.cargo/bin/:$PATH' >> $GITHUB_ENV @@ -383,7 +383,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-elixir@v1 with: - otp-version: '23.2.7' + otp-version: '23.3.4.1' elixir-version: '1.11.3' - run: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y - run: echo 'export PATH=~/.cargo/bin/:$PATH' >> $GITHUB_ENV diff --git a/CHANGELOG.md b/CHANGELOG.md index e603767f1b..468e4609cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ - [#4167](https://github.com/blockscout/blockscout/pull/4167) - Deduplicate block numbers in acquire_blocks function - [#4149](https://github.com/blockscout/blockscout/pull/4149) - Exclude smart_contract_additional_sources from JSON encoding in address schema - [#4137](https://github.com/blockscout/blockscout/pull/4137) - Get token balance query improvement +- [#4129](https://github.com/blockscout/blockscout/pull/4129) - Speedup procedure of finding missing block numbers for catchup fetcher - [#4038](https://github.com/blockscout/blockscout/pull/4038) - Add clause for abi_decode_address_output/1 when is_nil(address) - [#3989](https://github.com/blockscout/blockscout/pull/3989), [4061](https://github.com/blockscout/blockscout/pull/4061) - Fixed bug that sometimes lead to incorrect ordering of token transfers - [#3946](https://github.com/blockscout/blockscout/pull/3946) - Get NFT metadata from URIs with status_code 301 diff --git a/apps/explorer/lib/explorer/application.ex b/apps/explorer/lib/explorer/application.ex index 3e3ceb1f48..bb4429ae7b 100644 --- a/apps/explorer/lib/explorer/application.ex +++ b/apps/explorer/lib/explorer/application.ex @@ -15,6 +15,7 @@ defmodule Explorer.Application do BlockNumber, Blocks, GasUsage, + MinMissingBlockNumber, NetVersion, TransactionCount, Transactions, @@ -60,7 +61,8 @@ defmodule Explorer.Application do con_cache_child_spec(RSK.cache_name(), ttl_check_interval: :timer.minutes(1), global_ttl: :timer.minutes(30)), Transactions, Accounts, - Uncles + Uncles, + MinMissingBlockNumber ] children = base_children ++ configurable_children() diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index 555fad1475..d8faf1dc0e 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -1604,6 +1604,9 @@ defmodule Explorer.Chain do ) Repo.one(query) || 0 + rescue + _ -> + 0 end @spec fetch_max_block_number() :: non_neg_integer @@ -1617,6 +1620,9 @@ defmodule Explorer.Chain do ) Repo.one(query) || 0 + rescue + _ -> + 0 end @spec fetch_count_consensus_block() :: non_neg_integer @@ -2473,7 +2479,7 @@ defmodule Explorer.Chain do select: last_fetched_counter.value ) - Repo.one!(query) || 0 + Repo.one!(query) || Decimal.new(0) end defp block_status({number, timestamp}) do @@ -2489,6 +2495,33 @@ defmodule Explorer.Chain do defp block_status(nil), do: {:error, :no_blocks} + def fetch_min_missing_block_cache do + max_block_number = BlockNumber.get_max() + + if max_block_number > 0 do + query = + from(b in Block, + right_join: + missing_range in fragment( + """ + (SELECT b1.number + FROM generate_series(0, (?)::integer) AS b1(number) + WHERE NOT EXISTS + (SELECT 1 FROM blocks b2 WHERE b2.number=b1.number AND b2.consensus)) + """, + ^max_block_number + ), + on: b.number == missing_range.number, + select: min(missing_range.number) + ) + + query + |> Repo.one() || 0 + else + 0 + end + end + @doc """ Calculates the ranges of missing consensus blocks in `range`. diff --git a/apps/explorer/lib/explorer/chain/cache/block_number.ex b/apps/explorer/lib/explorer/chain/cache/block_number.ex index fb5f1b515a..90ee4892ee 100644 --- a/apps/explorer/lib/explorer/chain/cache/block_number.ex +++ b/apps/explorer/lib/explorer/chain/cache/block_number.ex @@ -34,7 +34,5 @@ defmodule Explorer.Chain.Cache.BlockNumber do :min -> Chain.fetch_min_block_number() :max -> Chain.fetch_max_block_number() end - rescue - _e -> 0 end end diff --git a/apps/explorer/lib/explorer/chain/cache/min_missing_block.ex b/apps/explorer/lib/explorer/chain/cache/min_missing_block.ex new file mode 100644 index 0000000000..47992ab845 --- /dev/null +++ b/apps/explorer/lib/explorer/chain/cache/min_missing_block.ex @@ -0,0 +1,52 @@ +defmodule Explorer.Chain.Cache.MinMissingBlockNumber do + @moduledoc """ + Caches min missing block number (break in the chain). + """ + + use GenServer + + alias Explorer.Chain + + @doc """ + Starts a process to periodically update the % of blocks indexed. + """ + @spec start_link(term()) :: GenServer.on_start() + def start_link(_) do + GenServer.start_link(__MODULE__, :ok, name: __MODULE__) + end + + @impl true + def init(args) do + Task.start_link(&fetch_min_missing_block/0) + + schedule_next_consolidation() + + {:ok, args} + end + + def fetch_min_missing_block do + result = Chain.fetch_min_missing_block_cache() + + if result > 0 do + params = %{ + counter_type: "min_missing_block_number", + value: result + } + + Chain.upsert_last_fetched_counter(params) + end + end + + defp schedule_next_consolidation do + Process.send_after(self(), :fetch_min_missing_block, :timer.minutes(20)) + end + + @impl true + def handle_info(:fetch_min_missing_block, state) do + fetch_min_missing_block() + + schedule_next_consolidation() + + {:noreply, state} + end +end diff --git a/apps/explorer/priv/repo/migrations/20210524165427_min_missing_block_number.exs b/apps/explorer/priv/repo/migrations/20210524165427_min_missing_block_number.exs new file mode 100644 index 0000000000..204ad4363a --- /dev/null +++ b/apps/explorer/priv/repo/migrations/20210524165427_min_missing_block_number.exs @@ -0,0 +1,12 @@ +defmodule Explorer.Repo.Migrations.MinMissingBlockNumber do + use Ecto.Migration + + def change do + insert_initial_genesis_block = """ + INSERT INTO last_fetched_counters (counter_type, value, inserted_at, updated_at) + VALUES ('min_missing_block_number', 0, NOW(), NOW()); + """ + + execute(insert_initial_genesis_block) + end +end diff --git a/apps/indexer/config/config.exs b/apps/indexer/config/config.exs index b79a4c36d0..b20a028fdb 100644 --- a/apps/indexer/config/config.exs +++ b/apps/indexer/config/config.exs @@ -35,7 +35,7 @@ config :indexer, String.to_integer(System.get_env("TOKEN_METADATA_UPDATE_INTERVAL") || "#{2 * 24 * 60 * 60}"), # bytes memory_limit: 1 <<< 30, - first_block: System.get_env("FIRST_BLOCK") || "0", + first_block: System.get_env("FIRST_BLOCK") || "", last_block: System.get_env("LAST_BLOCK") || "" config :indexer, Indexer.Fetcher.PendingTransaction.Supervisor, diff --git a/apps/indexer/lib/indexer/block/catchup/fetcher.ex b/apps/indexer/lib/indexer/block/catchup/fetcher.ex index 16dc05b979..c6f892f913 100644 --- a/apps/indexer/lib/indexer/block/catchup/fetcher.ex +++ b/apps/indexer/lib/indexer/block/catchup/fetcher.ex @@ -340,8 +340,16 @@ defmodule Indexer.Block.Catchup.Fetcher do string_value = Application.get_env(:indexer, :first_block) case Integer.parse(string_value) do - {integer, ""} -> integer - _ -> 0 + {integer, ""} -> + integer + + _ -> + min_missing_block_number = + "min_missing_block_number" + |> Chain.get_last_fetched_counter() + |> Decimal.to_integer() + + min_missing_block_number end end