From 83e848d5a4b2dc1062884bc412d9c3492b93aa09 Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Tue, 23 Jul 2024 15:24:08 +0300 Subject: [PATCH] perf: Reduce LookUpSmartContractSourcesOnDemand fetcher footprint (#10457) * Reduce LookUpSmartContractSourcesOnDemand fetcher footprint * LookUpSmartContractSourcesOnDemand: reduce input payload to several params * Remove need_to_check_and_partially_verified and eligibility_for_sources_fetching from state --- apps/explorer/lib/explorer/chain.ex | 21 ++- ...ook_up_smart_contract_sources_on_demand.ex | 141 ++++++++++++------ .../lib/explorer/chain/smart_contract.ex | 4 +- .../smart_contract/solidity/publish_helper.ex | 4 +- 4 files changed, 117 insertions(+), 53 deletions(-) diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index 30ecb22f92..f99aa779b1 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -1167,11 +1167,20 @@ defmodule Explorer.Chain do %{smart_contract: smart_contract} -> if smart_contract do CheckBytecodeMatchingOnDemand.trigger_check(address_result, smart_contract) - LookUpSmartContractSourcesOnDemand.trigger_fetch(address_result, smart_contract) + + LookUpSmartContractSourcesOnDemand.trigger_fetch( + to_string(address_result.hash), + address_result.contract_code, + smart_contract + ) SmartContract.check_and_update_constructor_args(address_result) else - LookUpSmartContractSourcesOnDemand.trigger_fetch(address_result, nil) + LookUpSmartContractSourcesOnDemand.trigger_fetch( + to_string(address_result.hash), + address_result.contract_code, + nil + ) {implementation_address_hashes, _} = Implementation.get_implementation( @@ -1190,7 +1199,13 @@ defmodule Explorer.Chain do end _ -> - LookUpSmartContractSourcesOnDemand.trigger_fetch(address_result, nil) + if address_result do + LookUpSmartContractSourcesOnDemand.trigger_fetch( + to_string(address_result.hash), + address_result.contract_code, + nil + ) + end address_result end diff --git a/apps/explorer/lib/explorer/chain/fetcher/look_up_smart_contract_sources_on_demand.ex b/apps/explorer/lib/explorer/chain/fetcher/look_up_smart_contract_sources_on_demand.ex index 47d4cb5920..1394f8aa0f 100644 --- a/apps/explorer/lib/explorer/chain/fetcher/look_up_smart_contract_sources_on_demand.ex +++ b/apps/explorer/lib/explorer/chain/fetcher/look_up_smart_contract_sources_on_demand.ex @@ -5,7 +5,7 @@ defmodule Explorer.Chain.Fetcher.LookUpSmartContractSourcesOnDemand do use GenServer - alias Explorer.Chain.{Address, Data, SmartContract} + alias Explorer.Chain.{Data, SmartContract} alias Explorer.Chain.Events.Publisher alias Explorer.SmartContract.EthBytecodeDBInterface alias Explorer.SmartContract.Solidity.Publisher, as: SolidityPublisher @@ -17,37 +17,41 @@ defmodule Explorer.Chain.Fetcher.LookUpSmartContractSourcesOnDemand do @cooldown_timeout 500 - def trigger_fetch(nil, _) do + def trigger_fetch(nil, _, _) do :ignore end - def trigger_fetch(address, %SmartContract{partially_verified: true}) do - GenServer.cast(__MODULE__, {:fetch, address}) + def trigger_fetch( + address_hash_string, + address_contract_code, + %SmartContract{partially_verified: true} + ) do + GenServer.cast(__MODULE__, {:check_eligibility, address_hash_string, address_contract_code, false}) end - def trigger_fetch(_address, %SmartContract{}) do + def trigger_fetch(_address_hash_string, _address_contract_code, %SmartContract{}) do :ignore end - def trigger_fetch(address, _) do - GenServer.cast(__MODULE__, {:fetch, address}) + def trigger_fetch(address_hash_string, address_contract_code, smart_contract) do + GenServer.cast(__MODULE__, {:check_eligibility, address_hash_string, address_contract_code, is_nil(smart_contract)}) end - defp fetch_sources(address, only_full?) do - Publisher.broadcast(%{eth_bytecode_db_lookup_started: [address.hash]}, :on_demand) + defp fetch_sources(address_hash_string, address_contract_code, only_full?) do + Publisher.broadcast(%{eth_bytecode_db_lookup_started: [address_hash_string]}, :on_demand) - creation_tx_input = contract_creation_input(address.hash) + creation_tx_input = contract_creation_input(address_hash_string) with {:ok, %{"sourceType" => type, "matchType" => match_type} = source} <- %{} - |> prepare_bytecode_for_microservice(creation_tx_input, Data.to_string(address.contract_code)) - |> EthBytecodeDBInterface.search_contract(address.hash), + |> prepare_bytecode_for_microservice(creation_tx_input, Data.to_string(address_contract_code)) + |> EthBytecodeDBInterface.search_contract(address_hash_string), :ok <- check_match_type(match_type, only_full?), - {:ok, _} <- process_contract_source(type, source, address.hash) do - Publisher.broadcast(%{smart_contract_was_verified: [address.hash]}, :on_demand) + {:ok, _} <- process_contract_source(type, source, address_hash_string) do + Publisher.broadcast(%{smart_contract_was_verified: [address_hash_string]}, :on_demand) else _ -> - Publisher.broadcast(%{smart_contract_was_not_verified: [address.hash]}, :on_demand) + Publisher.broadcast(%{smart_contract_was_not_verified: [address_hash_string]}, :on_demand) false end end @@ -73,26 +77,47 @@ defmodule Explorer.Chain.Fetcher.LookUpSmartContractSourcesOnDemand do end @impl true - def handle_cast({:fetch, address}, %{current_concurrency: counter, max_concurrency: max_concurrency} = state) + def handle_cast({:check_eligibility, address_hash_string, address_contract_code, nil_smart_contract?}, state) do + check_eligibility_for_sources_fetching(address_hash_string, address_contract_code, nil_smart_contract?, state) + end + + @impl true + def handle_cast( + {:fetch, address_hash_string, address_contract_code, need_to_check_and_partially_verified?}, + %{current_concurrency: counter, max_concurrency: max_concurrency} = state + ) when counter < max_concurrency do - handle_fetch_request(address, state) + handle_fetch_request(address_hash_string, address_contract_code, need_to_check_and_partially_verified?, state) end @impl true - def handle_cast({:fetch, _address} = request, %{current_concurrency: _counter} = state) do + def handle_cast( + {:fetch, _address_hash_string, _address_contract_code, _need_to_check_and_partially_verified?} = request, + %{current_concurrency: _counter} = state + ) do Process.send_after(self(), request, @cooldown_timeout) - {:noreply, state} end @impl true - def handle_info({:fetch, address}, %{current_concurrency: counter, max_concurrency: max_concurrency} = state) + def handle_info({:check_eligibility, address_hash_string, address_contract_code, nil_smart_contract?}, state) do + check_eligibility_for_sources_fetching(address_hash_string, address_contract_code, nil_smart_contract?, state) + end + + @impl true + def handle_info( + {:fetch, address_hash_string, address_contract_code, need_to_check_and_partially_verified?}, + %{current_concurrency: counter, max_concurrency: max_concurrency} = state + ) when counter < max_concurrency do - handle_fetch_request(address, state) + handle_fetch_request(address_hash_string, address_contract_code, need_to_check_and_partially_verified?, state) end @impl true - def handle_info({:fetch, _address} = request, state) do + def handle_info( + {:fetch, _address_hash_string, _address_contract_code, _need_to_check_and_partially_verified?} = request, + state + ) do Process.send_after(self(), request, @cooldown_timeout) {:noreply, state} end @@ -108,10 +133,10 @@ defmodule Explorer.Chain.Fetcher.LookUpSmartContractSourcesOnDemand do {:noreply, %{state | current_concurrency: counter - 1}} end - defp partially_verified?(%Address{smart_contract: nil}), do: nil + defp partially_verified?(_address_hash_string, true), do: nil - defp partially_verified?(%Address{hash: hash}) do - SmartContract.select_partially_verified_by_address_hash(hash) + defp partially_verified?(address_hash_string, _nil_smart_contract?) do + SmartContract.select_partially_verified_by_address_hash(address_hash_string) end defp check_interval(address_string) do @@ -129,16 +154,16 @@ defmodule Explorer.Chain.Fetcher.LookUpSmartContractSourcesOnDemand do end end - def process_contract_source("SOLIDITY", source, address_hash) do - SolidityPublisher.process_rust_verifier_response(source, address_hash, %{}, true, true, true) + def process_contract_source("SOLIDITY", source, address_hash_string) do + SolidityPublisher.process_rust_verifier_response(source, address_hash_string, %{}, true, true, true) end - def process_contract_source("VYPER", source, address_hash) do - VyperPublisher.process_rust_verifier_response(source, address_hash, %{}, true, true, true) + def process_contract_source("VYPER", source, address_hash_string) do + VyperPublisher.process_rust_verifier_response(source, address_hash_string, %{}, true, true, true) end - def process_contract_source("YUL", source, address_hash) do - SolidityPublisher.process_rust_verifier_response(source, address_hash, %{}, true, true, true) + def process_contract_source("YUL", source, address_hash_string) do + SolidityPublisher.process_rust_verifier_response(source, address_hash_string, %{}, true, true, true) end def process_contract_source(_, _source, _address_hash), do: false @@ -146,25 +171,49 @@ defmodule Explorer.Chain.Fetcher.LookUpSmartContractSourcesOnDemand do defp check_match_type("PARTIAL", true), do: :full_match_required defp check_match_type(_, _), do: :ok - defp handle_fetch_request(address, %{current_concurrency: counter} = state) do - need_to_check_and_partially_verified? = - check_interval(to_lowercase_string(address.hash)) && partially_verified?(address) - - diff = - if is_nil(need_to_check_and_partially_verified?) || need_to_check_and_partially_verified? do - Task.Supervisor.async_nolink(Explorer.GenesisDataTaskSupervisor, fn -> - fetch_sources(address, need_to_check_and_partially_verified?) - end) + defp handle_fetch_request( + address_hash_string, + address_contract_code, + need_to_check_and_partially_verified?, + %{ + current_concurrency: counter + } = state + ) do + Task.Supervisor.async_nolink(Explorer.GenesisDataTaskSupervisor, fn -> + fetch_sources(address_hash_string, address_contract_code, need_to_check_and_partially_verified?) + end) - :ets.insert(@cache_name, {to_lowercase_string(address.hash), DateTime.utc_now()}) + :ets.insert(@cache_name, {to_lowercase_string(address_hash_string), DateTime.utc_now()}) - 1 - else - 0 - end + diff = 1 {:noreply, %{state | current_concurrency: counter + diff}} end - defp to_lowercase_string(hash), do: hash |> to_string() |> String.downcase() + defp eligible_for_sources_fetching?(need_to_check_and_partially_verified?) do + is_nil(need_to_check_and_partially_verified?) || need_to_check_and_partially_verified? + end + + @spec stale_and_partially_verified?(String.t(), boolean()) :: boolean() | nil + defp stale_and_partially_verified?(address_hash_string, nil_smart_contract?) do + check_interval(to_lowercase_string(address_hash_string)) && + partially_verified?(address_hash_string, nil_smart_contract?) + end + + defp check_eligibility_for_sources_fetching(address_hash_string, address_contract_code, nil_smart_contract?, state) do + need_to_check_and_partially_verified? = stale_and_partially_verified?(address_hash_string, nil_smart_contract?) + + eligibility_for_sources_fetching = eligible_for_sources_fetching?(need_to_check_and_partially_verified?) + + if eligibility_for_sources_fetching do + GenServer.cast( + __MODULE__, + {:fetch, address_hash_string, address_contract_code, need_to_check_and_partially_verified?} + ) + end + + {:noreply, state} + end + + defp to_lowercase_string(address_hash_string), do: address_hash_string |> String.downcase() end diff --git a/apps/explorer/lib/explorer/chain/smart_contract.ex b/apps/explorer/lib/explorer/chain/smart_contract.ex index 9d44ed40bf..71a2a0064f 100644 --- a/apps/explorer/lib/explorer/chain/smart_contract.ex +++ b/apps/explorer/lib/explorer/chain/smart_contract.ex @@ -513,11 +513,11 @@ defmodule Explorer.Chain.SmartContract do Returns SmartContract by the given smart-contract address hash, if it is partially verified """ @spec select_partially_verified_by_address_hash(binary() | Hash.t(), keyword) :: boolean() | nil - def select_partially_verified_by_address_hash(address_hash, options \\ []) do + def select_partially_verified_by_address_hash(address_hash_string, options \\ []) do query = from( smart_contract in __MODULE__, - where: smart_contract.address_hash == ^address_hash, + where: smart_contract.address_hash == ^address_hash_string, select: smart_contract.partially_verified ) diff --git a/apps/explorer/lib/explorer/smart_contract/solidity/publish_helper.ex b/apps/explorer/lib/explorer/smart_contract/solidity/publish_helper.ex index 4af55016fc..c452a97f7e 100644 --- a/apps/explorer/lib/explorer/smart_contract/solidity/publish_helper.ex +++ b/apps/explorer/lib/explorer/smart_contract/solidity/publish_helper.ex @@ -4,9 +4,9 @@ defmodule Explorer.SmartContract.Solidity.PublishHelper do """ alias Ecto.Changeset - alias Explorer.Chain.{Address, SmartContract} alias Explorer.Chain.Events.Publisher, as: EventsPublisher alias Explorer.Chain.Fetcher.LookUpSmartContractSourcesOnDemand + alias Explorer.Chain.SmartContract alias Explorer.SmartContract.Solidity.Publisher alias Explorer.ThirdPartyIntegrations.Sourcify @@ -149,7 +149,7 @@ defmodule Explorer.SmartContract.Solidity.PublishHelper do def check_and_verify(address_hash_string) do if Application.get_env(:explorer, Explorer.SmartContract.RustVerifierInterfaceBehaviour)[:eth_bytecode_db?] do - LookUpSmartContractSourcesOnDemand.trigger_fetch(%Address{hash: address_hash_string}, nil) + LookUpSmartContractSourcesOnDemand.trigger_fetch(address_hash_string, nil, nil) else if Application.get_env(:explorer, Explorer.ThirdPartyIntegrations.Sourcify)[:enabled] do check_by_address_in_sourcify(