From 8780fdfdacda825b9bbec69ddd48a8feeff1b499 Mon Sep 17 00:00:00 2001 From: Nikita Pozdniakov Date: Thu, 16 Feb 2023 22:57:40 +0300 Subject: [PATCH] Add FetchValidatorInfoOnDemand fetcher --- apps/explorer/config/config.exs | 4 +- apps/explorer/lib/explorer/application.ex | 3 +- .../lib/explorer/application/constants.ex | 46 +++++++++++++ apps/explorer/lib/explorer/chain.ex | 2 +- .../lib/explorer/chain/block/reward.ex | 37 +++++++++- .../check_bytecode_matching_on_demand.ex | 23 +------ .../fetcher/fetch_validator_info_on_demand.ex | 69 +++++++++++++++++++ apps/explorer/lib/explorer/chain/validator.ex | 56 +++++++++++++++ ...17_add_validators_and_constants_tables.exs | 21 ++++++ .../indexer/fetcher/first_trace_on_demand.ex | 19 +---- 10 files changed, 237 insertions(+), 43 deletions(-) create mode 100644 apps/explorer/lib/explorer/application/constants.ex rename apps/explorer/lib/explorer/chain/{checker => fetcher}/check_bytecode_matching_on_demand.ex (77%) create mode 100644 apps/explorer/lib/explorer/chain/fetcher/fetch_validator_info_on_demand.ex create mode 100644 apps/explorer/lib/explorer/chain/validator.ex create mode 100644 apps/explorer/priv/repo/migrations/20230214104917_add_validators_and_constants_tables.exs diff --git a/apps/explorer/config/config.exs b/apps/explorer/config/config.exs index 4f6e5649c9..c7b97b016e 100644 --- a/apps/explorer/config/config.exs +++ b/apps/explorer/config/config.exs @@ -106,7 +106,9 @@ config :explorer, Explorer.Counters.BlockPriorityFeeCounter, config :explorer, Explorer.TokenTransferTokenIdMigration.Supervisor, enabled: true -config :explorer, Explorer.Chain.Checker.CheckBytecodeMatchingOnDemand, enabled: true +config :explorer, Explorer.Chain.Fetcher.CheckBytecodeMatchingOnDemand, enabled: true + +config :explorer, Explorer.Chain.Fetcher.FetchValidatorInfoOnDemand, enabled: true config :explorer, Explorer.Chain.Cache.GasUsage, enabled: System.get_env("CACHE_ENABLE_TOTAL_GAS_USAGE_COUNTER") == "true" diff --git a/apps/explorer/lib/explorer/application.ex b/apps/explorer/lib/explorer/application.ex index d7c3f7875d..7d9d66c576 100644 --- a/apps/explorer/lib/explorer/application.ex +++ b/apps/explorer/lib/explorer/application.ex @@ -109,7 +109,8 @@ defmodule Explorer.Application do configure(Explorer.Tags.AddressTag.Cataloger), configure(MinMissingBlockNumber), configure(TokenTransferTokenIdMigration.Supervisor), - configure(Explorer.Chain.Checker.CheckBytecodeMatchingOnDemand) + configure(Explorer.Chain.Fetcher.CheckBytecodeMatchingOnDemand), + configure(Explorer.Chain.Fetcher.FetchValidatorInfoOnDemand) ] |> List.flatten() end diff --git a/apps/explorer/lib/explorer/application/constants.ex b/apps/explorer/lib/explorer/application/constants.ex new file mode 100644 index 0000000000..52abd80894 --- /dev/null +++ b/apps/explorer/lib/explorer/application/constants.ex @@ -0,0 +1,46 @@ +defmodule Explorer.Application.Constants do + @moduledoc """ + Tracks some kv info + """ + + use Explorer.Schema + alias Explorer.Repo + + @keys_manager_contract_address_key "keys_manager_contract_address" + + @primary_key false + schema "constants" do + field(:key, :string, primary_key: true) + field(:value, :string) + + timestamps() + end + + def changeset(attrs) do + %__MODULE__{} + |> changeset(attrs) + end + + @required_attrs ~w(key value)a + def changeset(%__MODULE__{} = constant, attrs) do + constant + |> cast(attrs, @required_attrs) + |> validate_required(@required_attrs) + end + + def get_constant_by_key(key) do + __MODULE__ + |> where([constant], constant.key == ^key) + |> Repo.one() + end + + def insert_keys_manager_contract_address(value) do + %{key: @keys_manager_contract_address_key, value: value} + |> changeset() + |> Repo.insert!() + end + + def get_keys_manager_contract_address do + get_constant_by_key(@keys_manager_contract_address_key) + end +end diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index fd833533e4..19d61e260a 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -80,7 +80,7 @@ defmodule Explorer.Chain do alias Explorer.Chain.Import.Runner alias Explorer.Chain.InternalTransaction.{CallType, Type} - alias Explorer.Chain.Checker.CheckBytecodeMatchingOnDemand + alias Explorer.Chain.Fetcher.CheckBytecodeMatchingOnDemand alias Explorer.Counters.{ AddressesCounter, diff --git a/apps/explorer/lib/explorer/chain/block/reward.ex b/apps/explorer/lib/explorer/chain/block/reward.ex index f3c9bfa05e..84a9da9084 100644 --- a/apps/explorer/lib/explorer/chain/block/reward.ex +++ b/apps/explorer/lib/explorer/chain/block/reward.ex @@ -5,9 +5,11 @@ defmodule Explorer.Chain.Block.Reward do use Explorer.Schema + alias Explorer.Application.Constants alias Explorer.Chain alias Explorer.Chain.Block.Reward.AddressType - alias Explorer.Chain.{Address, Block, Hash, Wei} + alias Explorer.Chain.{Address, Block, Hash, Wei, Validator} + alias Explorer.Chain.Fetcher.FetchValidatorInfoOnDemand alias Explorer.{PagingOptions, Repo} alias Explorer.SmartContract.Reader @@ -152,6 +154,39 @@ defmodule Explorer.Chain.Block.Reward do end end + def get_validator_payout_key_by_mining_from_db(mining_key) do + contract_address_from_db = Constants.get_keys_manager_contract_address() + + contract_address_from_env = + Application.get_env(:explorer, Explorer.Chain.Block.Reward, %{})[:keys_manager_contract_address] + + cond do + is_nil(contract_address_from_env) -> + %{is_validator: nil, payout_key: mining_key} + + is_nil(contract_address_from_db) -> + FetchValidatorInfoOnDemand.trigger_fetch(mining_key) + %{is_validator: nil, payout_key: mining_key} + + contract_address_from_db.value |> String.downcase() == contract_address_from_env |> String.downcase() -> + FetchValidatorInfoOnDemand.trigger_fetch(mining_key) + validator = Validator.get_validator_by_address_hash(mining_key) + is_validator = validator && validator.is_validator + + with {:is_validator, true} <- {:is_validator, is_validator}, + false <- is_nil(validator.payout_key_hash) do + %{is_validator: is_validator, payout_key: validator.payout_key_hash} + else + _ -> + %{is_validator: is_validator, payout_key: mining_key} + end + + true -> + FetchValidatorInfoOnDemand.trigger_fetch(mining_key) + %{is_validator: nil, payout_key: mining_key} + end + end + def get_validator_payout_key_by_mining(mining_key) do is_validator = is_validator(mining_key) diff --git a/apps/explorer/lib/explorer/chain/checker/check_bytecode_matching_on_demand.ex b/apps/explorer/lib/explorer/chain/fetcher/check_bytecode_matching_on_demand.ex similarity index 77% rename from apps/explorer/lib/explorer/chain/checker/check_bytecode_matching_on_demand.ex rename to apps/explorer/lib/explorer/chain/fetcher/check_bytecode_matching_on_demand.ex index 15e5a9a8c9..ec28298b00 100644 --- a/apps/explorer/lib/explorer/chain/checker/check_bytecode_matching_on_demand.ex +++ b/apps/explorer/lib/explorer/chain/fetcher/check_bytecode_matching_on_demand.ex @@ -1,4 +1,4 @@ -defmodule Explorer.Chain.Checker.CheckBytecodeMatchingOnDemand do +defmodule Explorer.Chain.Fetcher.CheckBytecodeMatchingOnDemand do @moduledoc """ On demand checker if bytecode written in BlockScout's DB equals to bytecode stored on node (only for verified contracts) """ @@ -8,16 +8,10 @@ defmodule Explorer.Chain.Checker.CheckBytecodeMatchingOnDemand do alias Ecto.Association.NotLoaded alias Ecto.Changeset alias Explorer.Repo - alias Explorer.Counters.Helper - - require Logger # seconds @check_bytecode_interval 86_400 - # cache needed to keep track of transactions which are already being processed - @cache_name :bytecode_matching_processing - def trigger_check(_address, %NotLoaded{}) do :ignore end @@ -62,8 +56,6 @@ defmodule Explorer.Chain.Checker.CheckBytecodeMatchingOnDemand do :error end end - - :ets.delete(@cache_name, to_string(address.hash)) end def start_link(_) do @@ -72,23 +64,12 @@ defmodule Explorer.Chain.Checker.CheckBytecodeMatchingOnDemand do @impl true def init(opts) do - Helper.create_cache_table(@cache_name) - {:ok, opts} end @impl true def handle_cast({:check, address}, state) do - hash_string = to_string(address.hash) - - case Helper.fetch_from_cache(hash_string, @cache_name) do - 0 -> - :ets.insert(@cache_name, {hash_string, 1}) - check_bytecode_matching(address) - - 1 -> - :ignore - end + check_bytecode_matching(address) {:noreply, state} end diff --git a/apps/explorer/lib/explorer/chain/fetcher/fetch_validator_info_on_demand.ex b/apps/explorer/lib/explorer/chain/fetcher/fetch_validator_info_on_demand.ex new file mode 100644 index 0000000000..495f614634 --- /dev/null +++ b/apps/explorer/lib/explorer/chain/fetcher/fetch_validator_info_on_demand.ex @@ -0,0 +1,69 @@ +defmodule Explorer.Chain.Fetcher.FetchValidatorInfoOnDemand do + @moduledoc """ + On demand fetcher info about validator + """ + + use GenServer + + alias Explorer.Application.Constants + alias Explorer.Chain.Block.Reward + alias Explorer.Chain.Cache.BlockNumber + alias Explorer.Chain.Validator + + def trigger_fetch(address_hash) do + GenServer.cast(__MODULE__, {:fetch_or_update, address_hash}) + end + + defp actualize_validator_info(address_hash) do + contract_address_from_db = Constants.get_keys_manager_contract_address() + + contract_address_from_env = + Application.get_env(:explorer, Explorer.Chain.Block.Reward, %{})[:keys_manager_contract_address] + + cond do + is_nil(contract_address_from_env) -> + :ignore + + is_nil(contract_address_from_db) -> + Validator.drop_all_validators() + Constants.insert_keys_manager_contract_address(contract_address_from_env) + fetch_and_store_validator_info(address_hash) + + String.downcase(contract_address_from_db.value) == contract_address_from_env |> String.downcase() -> + fetch_and_store_validator_info(address_hash) + + true -> + Validator.drop_all_validators() + Constants.insert_keys_manager_contract_address(contract_address_from_env) + fetch_and_store_validator_info(address_hash) + end + end + + defp fetch_and_store_validator_info(validator_address) do + validator = Validator.get_validator_by_address_hash(validator_address) + %{is_validator: is_validator, payout_key: payout_key} = Reward.get_validator_payout_key_by_mining(validator_address) + + Validator.insert_or_update(validator, %{ + address_hash: validator_address, + is_validator: is_validator, + payout_key_hash: payout_key, + last_block_updated_at: BlockNumber.get_max() + }) + end + + def start_link(_) do + GenServer.start_link(__MODULE__, :ok, name: __MODULE__) + end + + @impl true + def init(opts) do + {:ok, opts} + end + + @impl true + def handle_cast({:fetch_or_update, address_hash}, state) do + actualize_validator_info(address_hash) + + {:noreply, state} + end +end diff --git a/apps/explorer/lib/explorer/chain/validator.ex b/apps/explorer/lib/explorer/chain/validator.ex new file mode 100644 index 0000000000..5897776d5e --- /dev/null +++ b/apps/explorer/lib/explorer/chain/validator.ex @@ -0,0 +1,56 @@ +defmodule Explorer.Chain.Validator do + @moduledoc """ + Tracks info about POA validator + """ + + use Explorer.Schema + alias Explorer.Chain.Hash.Address + alias Explorer.Repo + + @primary_key false + schema "validators" do + field(:address_hash, Address, primary_key: true) + field(:is_validator, :boolean) + field(:payout_key_hash, Address) + field(:last_block_updated_at, :integer) + + timestamps() + end + + def insert_or_update(nil, attrs) do + attrs + |> changeset() + |> Repo.insert() + end + + def insert_or_update(validator, attrs) do + validator + |> changeset(attrs) + |> Repo.update() + end + + def changeset(attrs) do + %__MODULE__{} + |> changeset(attrs) + end + + @required_attrs ~w(address_hash)a + @optional_attrs ~w(is_validator payout_key_hash last_block_updated_at)a + def changeset(%__MODULE__{} = constant, attrs) do + constant + |> cast(attrs, @required_attrs ++ @optional_attrs) + |> validate_required(@required_attrs) + |> unique_constraint(:address_hash) + end + + def get_validator_by_address_hash(address_hash) do + __MODULE__ + |> where([validator], validator.address_hash == ^address_hash) + |> Repo.one() + end + + def drop_all_validators() do + __MODULE__ + |> Repo.delete_all() + end +end diff --git a/apps/explorer/priv/repo/migrations/20230214104917_add_validators_and_constants_tables.exs b/apps/explorer/priv/repo/migrations/20230214104917_add_validators_and_constants_tables.exs new file mode 100644 index 0000000000..690f039d65 --- /dev/null +++ b/apps/explorer/priv/repo/migrations/20230214104917_add_validators_and_constants_tables.exs @@ -0,0 +1,21 @@ +defmodule Explorer.Repo.Migrations.AddValidatorsAndConstantsTables do + use Ecto.Migration + + def change do + create table(:constants, primary_key: false) do + add(:key, :string, primary_key: true, null: false) + add(:value, :string) + + timestamps() + end + + create table(:validators, primary_key: false) do + add(:address_hash, :bytea, primary_key: true, null: false) + add(:is_validator, :boolean) + add(:payout_key_hash, :bytea) + add(:last_block_updated_at, :bigint) + + timestamps() + end + end +end diff --git a/apps/indexer/lib/indexer/fetcher/first_trace_on_demand.ex b/apps/indexer/lib/indexer/fetcher/first_trace_on_demand.ex index 04ba5873d4..10bbc181dd 100644 --- a/apps/indexer/lib/indexer/fetcher/first_trace_on_demand.ex +++ b/apps/indexer/lib/indexer/fetcher/first_trace_on_demand.ex @@ -9,13 +9,9 @@ defmodule Indexer.Fetcher.FirstTraceOnDemand do alias Explorer.Chain alias Explorer.Chain.Import alias Explorer.Chain.Import.Runner.InternalTransactions - alias Explorer.Counters.Helper require Logger - # cache needed to keep track of transactions which are already being processed - @cache_name :first_traces_processing - def trigger_fetch(transaction) do GenServer.cast(__MODULE__, {:fetch, transaction}) end @@ -52,8 +48,6 @@ defmodule Indexer.Fetcher.FirstTraceOnDemand do :ignore -> :ignore end - - :ets.delete(@cache_name, hash_string) end def start_link([init_opts, server_opts]) do @@ -62,23 +56,12 @@ defmodule Indexer.Fetcher.FirstTraceOnDemand do @impl true def init(json_rpc_named_arguments) do - Helper.create_cache_table(@cache_name) - {:ok, %{json_rpc_named_arguments: json_rpc_named_arguments}} end @impl true def handle_cast({:fetch, transaction}, state) do - hash_string = to_string(transaction.hash) - - case Helper.fetch_from_cache(hash_string, @cache_name) do - 0 -> - :ets.insert(@cache_name, {hash_string, 1}) - fetch_first_trace(transaction, state) - - 1 -> - :ignore - end + fetch_first_trace(transaction, state) {:noreply, state} end