perf: Reduce LookUpSmartContractSourcesOnDemand fetcher footprint (#10457)

* Reduce LookUpSmartContractSourcesOnDemand fetcher footprint * LookUpSmartContractSourcesOnDemand: reduce input payload to several params * Remove need_to_check_and_partially_verified and eligibility_for_sources_fetching from state
4 months ago · 83e848d5a4
parent b140e9f5b9
commit 83e848d5a4
4 changed files with 117 additions and 53 deletions
--- a/apps/explorer/lib/explorer/chain.ex
+++ b/apps/explorer/lib/explorer/chain.ex
@ -1167,11 +1167,20 @@ defmodule Explorer.Chain do
        %{smart_contract: smart_contract} ->
          if smart_contract do
            CheckBytecodeMatchingOnDemand.trigger_check(address_result, smart_contract)
-            LookUpSmartContractSourcesOnDemand.trigger_fetch(address_result, smart_contract)
+
+            LookUpSmartContractSourcesOnDemand.trigger_fetch(
+              to_string(address_result.hash),
+              address_result.contract_code,
+              smart_contract
+            )

            SmartContract.check_and_update_constructor_args(address_result)
          else
-            LookUpSmartContractSourcesOnDemand.trigger_fetch(address_result, nil)
+            LookUpSmartContractSourcesOnDemand.trigger_fetch(
+              to_string(address_result.hash),
+              address_result.contract_code,
+              nil
+            )

            {implementation_address_hashes, _} =
              Implementation.get_implementation(
@ -1190,7 +1199,13 @@ defmodule Explorer.Chain do
          end

        _ ->
-          LookUpSmartContractSourcesOnDemand.trigger_fetch(address_result, nil)
+          if address_result do
+            LookUpSmartContractSourcesOnDemand.trigger_fetch(
+              to_string(address_result.hash),
+              address_result.contract_code,
+              nil
+            )
+          end

          address_result
      end
--- a/apps/explorer/lib/explorer/chain/fetcher/look_up_smart_contract_sources_on_demand.ex
+++ b/apps/explorer/lib/explorer/chain/fetcher/look_up_smart_contract_sources_on_demand.ex
@ -5,7 +5,7 @@ defmodule Explorer.Chain.Fetcher.LookUpSmartContractSourcesOnDemand do

  use GenServer

-  alias Explorer.Chain.{Address, Data, SmartContract}
+  alias Explorer.Chain.{Data, SmartContract}
  alias Explorer.Chain.Events.Publisher
  alias Explorer.SmartContract.EthBytecodeDBInterface
  alias Explorer.SmartContract.Solidity.Publisher, as: SolidityPublisher
@ -17,37 +17,41 @@ defmodule Explorer.Chain.Fetcher.LookUpSmartContractSourcesOnDemand do

  @cooldown_timeout 500

-  def trigger_fetch(nil, _) do
+  def trigger_fetch(nil, _, _) do
    :ignore
  end

-  def trigger_fetch(address, %SmartContract{partially_verified: true}) do
-    GenServer.cast(__MODULE__, {:fetch, address})
+  def trigger_fetch(
+        address_hash_string,
+        address_contract_code,
+        %SmartContract{partially_verified: true}
+      ) do
+    GenServer.cast(__MODULE__, {:check_eligibility, address_hash_string, address_contract_code, false})
  end

-  def trigger_fetch(_address, %SmartContract{}) do
+  def trigger_fetch(_address_hash_string, _address_contract_code, %SmartContract{}) do
    :ignore
  end

-  def trigger_fetch(address, _) do
-    GenServer.cast(__MODULE__, {:fetch, address})
+  def trigger_fetch(address_hash_string, address_contract_code, smart_contract) do
+    GenServer.cast(__MODULE__, {:check_eligibility, address_hash_string, address_contract_code, is_nil(smart_contract)})
  end

-  defp fetch_sources(address, only_full?) do
-    Publisher.broadcast(%{eth_bytecode_db_lookup_started: [address.hash]}, :on_demand)
+  defp fetch_sources(address_hash_string, address_contract_code, only_full?) do
+    Publisher.broadcast(%{eth_bytecode_db_lookup_started: [address_hash_string]}, :on_demand)

-    creation_tx_input = contract_creation_input(address.hash)
+    creation_tx_input = contract_creation_input(address_hash_string)

    with {:ok, %{"sourceType" => type, "matchType" => match_type} = source} <-
           %{}
-           |> prepare_bytecode_for_microservice(creation_tx_input, Data.to_string(address.contract_code))
-           |> EthBytecodeDBInterface.search_contract(address.hash),
+           |> prepare_bytecode_for_microservice(creation_tx_input, Data.to_string(address_contract_code))
+           |> EthBytecodeDBInterface.search_contract(address_hash_string),
         :ok <- check_match_type(match_type, only_full?),
-         {:ok, _} <- process_contract_source(type, source, address.hash) do
-      Publisher.broadcast(%{smart_contract_was_verified: [address.hash]}, :on_demand)
+         {:ok, _} <- process_contract_source(type, source, address_hash_string) do
+      Publisher.broadcast(%{smart_contract_was_verified: [address_hash_string]}, :on_demand)
    else
      _ ->
-        Publisher.broadcast(%{smart_contract_was_not_verified: [address.hash]}, :on_demand)
+        Publisher.broadcast(%{smart_contract_was_not_verified: [address_hash_string]}, :on_demand)
        false
    end
  end
@ -73,26 +77,47 @@ defmodule Explorer.Chain.Fetcher.LookUpSmartContractSourcesOnDemand do
  end

  @impl true
-  def handle_cast({:fetch, address}, %{current_concurrency: counter, max_concurrency: max_concurrency} = state)
+  def handle_cast({:check_eligibility, address_hash_string, address_contract_code, nil_smart_contract?}, state) do
+    check_eligibility_for_sources_fetching(address_hash_string, address_contract_code, nil_smart_contract?, state)
+  end
+
+  @impl true
+  def handle_cast(
+        {:fetch, address_hash_string, address_contract_code, need_to_check_and_partially_verified?},
+        %{current_concurrency: counter, max_concurrency: max_concurrency} = state
+      )
      when counter < max_concurrency do
-    handle_fetch_request(address, state)
+    handle_fetch_request(address_hash_string, address_contract_code, need_to_check_and_partially_verified?, state)
  end

  @impl true
-  def handle_cast({:fetch, _address} = request, %{current_concurrency: _counter} = state) do
+  def handle_cast(
+        {:fetch, _address_hash_string, _address_contract_code, _need_to_check_and_partially_verified?} = request,
+        %{current_concurrency: _counter} = state
+      ) do
    Process.send_after(self(), request, @cooldown_timeout)
-
    {:noreply, state}
  end

  @impl true
-  def handle_info({:fetch, address}, %{current_concurrency: counter, max_concurrency: max_concurrency} = state)
+  def handle_info({:check_eligibility, address_hash_string, address_contract_code, nil_smart_contract?}, state) do
+    check_eligibility_for_sources_fetching(address_hash_string, address_contract_code, nil_smart_contract?, state)
+  end
+
+  @impl true
+  def handle_info(
+        {:fetch, address_hash_string, address_contract_code, need_to_check_and_partially_verified?},
+        %{current_concurrency: counter, max_concurrency: max_concurrency} = state
+      )
      when counter < max_concurrency do
-    handle_fetch_request(address, state)
+    handle_fetch_request(address_hash_string, address_contract_code, need_to_check_and_partially_verified?, state)
  end

  @impl true
-  def handle_info({:fetch, _address} = request, state) do
+  def handle_info(
+        {:fetch, _address_hash_string, _address_contract_code, _need_to_check_and_partially_verified?} = request,
+        state
+      ) do
    Process.send_after(self(), request, @cooldown_timeout)
    {:noreply, state}
  end
@ -108,10 +133,10 @@ defmodule Explorer.Chain.Fetcher.LookUpSmartContractSourcesOnDemand do
    {:noreply, %{state | current_concurrency: counter - 1}}
  end

-  defp partially_verified?(%Address{smart_contract: nil}), do: nil
+  defp partially_verified?(_address_hash_string, true), do: nil

-  defp partially_verified?(%Address{hash: hash}) do
-    SmartContract.select_partially_verified_by_address_hash(hash)
+  defp partially_verified?(address_hash_string, _nil_smart_contract?) do
+    SmartContract.select_partially_verified_by_address_hash(address_hash_string)
  end

  defp check_interval(address_string) do
@ -129,16 +154,16 @@ defmodule Explorer.Chain.Fetcher.LookUpSmartContractSourcesOnDemand do
    end
  end

-  def process_contract_source("SOLIDITY", source, address_hash) do
-    SolidityPublisher.process_rust_verifier_response(source, address_hash, %{}, true, true, true)
+  def process_contract_source("SOLIDITY", source, address_hash_string) do
+    SolidityPublisher.process_rust_verifier_response(source, address_hash_string, %{}, true, true, true)
  end

-  def process_contract_source("VYPER", source, address_hash) do
-    VyperPublisher.process_rust_verifier_response(source, address_hash, %{}, true, true, true)
+  def process_contract_source("VYPER", source, address_hash_string) do
+    VyperPublisher.process_rust_verifier_response(source, address_hash_string, %{}, true, true, true)
  end

-  def process_contract_source("YUL", source, address_hash) do
-    SolidityPublisher.process_rust_verifier_response(source, address_hash, %{}, true, true, true)
+  def process_contract_source("YUL", source, address_hash_string) do
+    SolidityPublisher.process_rust_verifier_response(source, address_hash_string, %{}, true, true, true)
  end

  def process_contract_source(_, _source, _address_hash), do: false
@ -146,25 +171,49 @@ defmodule Explorer.Chain.Fetcher.LookUpSmartContractSourcesOnDemand do
  defp check_match_type("PARTIAL", true), do: :full_match_required
  defp check_match_type(_, _), do: :ok

-  defp handle_fetch_request(address, %{current_concurrency: counter} = state) do
-    need_to_check_and_partially_verified? =
-      check_interval(to_lowercase_string(address.hash)) && partially_verified?(address)
-
-    diff =
-      if is_nil(need_to_check_and_partially_verified?) || need_to_check_and_partially_verified? do
-        Task.Supervisor.async_nolink(Explorer.GenesisDataTaskSupervisor, fn ->
-          fetch_sources(address, need_to_check_and_partially_verified?)
-        end)
+  defp handle_fetch_request(
+         address_hash_string,
+         address_contract_code,
+         need_to_check_and_partially_verified?,
+         %{
+           current_concurrency: counter
+         } = state
+       ) do
+    Task.Supervisor.async_nolink(Explorer.GenesisDataTaskSupervisor, fn ->
+      fetch_sources(address_hash_string, address_contract_code, need_to_check_and_partially_verified?)
+    end)

-        :ets.insert(@cache_name, {to_lowercase_string(address.hash), DateTime.utc_now()})
+    :ets.insert(@cache_name, {to_lowercase_string(address_hash_string), DateTime.utc_now()})

-        1
-      else
-        0
-      end
+    diff = 1

    {:noreply, %{state | current_concurrency: counter + diff}}
  end

-  defp to_lowercase_string(hash), do: hash |> to_string() |> String.downcase()
+  defp eligible_for_sources_fetching?(need_to_check_and_partially_verified?) do
+    is_nil(need_to_check_and_partially_verified?) || need_to_check_and_partially_verified?
+  end
+
+  @spec stale_and_partially_verified?(String.t(), boolean()) :: boolean() | nil
+  defp stale_and_partially_verified?(address_hash_string, nil_smart_contract?) do
+    check_interval(to_lowercase_string(address_hash_string)) &&
+      partially_verified?(address_hash_string, nil_smart_contract?)
+  end
+
+  defp check_eligibility_for_sources_fetching(address_hash_string, address_contract_code, nil_smart_contract?, state) do
+    need_to_check_and_partially_verified? = stale_and_partially_verified?(address_hash_string, nil_smart_contract?)
+
+    eligibility_for_sources_fetching = eligible_for_sources_fetching?(need_to_check_and_partially_verified?)
+
+    if eligibility_for_sources_fetching do
+      GenServer.cast(
+        __MODULE__,
+        {:fetch, address_hash_string, address_contract_code, need_to_check_and_partially_verified?}
+      )
+    end
+
+    {:noreply, state}
+  end
+
+  defp to_lowercase_string(address_hash_string), do: address_hash_string |> String.downcase()
 end
--- a/apps/explorer/lib/explorer/chain/smart_contract.ex
+++ b/apps/explorer/lib/explorer/chain/smart_contract.ex
@ -513,11 +513,11 @@ defmodule Explorer.Chain.SmartContract do
  Returns SmartContract by the given smart-contract address hash, if it is partially verified
  """
  @spec select_partially_verified_by_address_hash(binary() | Hash.t(), keyword) :: boolean() | nil
-  def select_partially_verified_by_address_hash(address_hash, options \\ []) do
+  def select_partially_verified_by_address_hash(address_hash_string, options \\ []) do
    query =
      from(
        smart_contract in __MODULE__,
-        where: smart_contract.address_hash == ^address_hash,
+        where: smart_contract.address_hash == ^address_hash_string,
        select: smart_contract.partially_verified
      )

--- a/apps/explorer/lib/explorer/smart_contract/solidity/publish_helper.ex
+++ b/apps/explorer/lib/explorer/smart_contract/solidity/publish_helper.ex
@ -4,9 +4,9 @@ defmodule Explorer.SmartContract.Solidity.PublishHelper do
  """

  alias Ecto.Changeset
-  alias Explorer.Chain.{Address, SmartContract}
  alias Explorer.Chain.Events.Publisher, as: EventsPublisher
  alias Explorer.Chain.Fetcher.LookUpSmartContractSourcesOnDemand
+  alias Explorer.Chain.SmartContract
  alias Explorer.SmartContract.Solidity.Publisher
  alias Explorer.ThirdPartyIntegrations.Sourcify

@ -149,7 +149,7 @@ defmodule Explorer.SmartContract.Solidity.PublishHelper do

  def check_and_verify(address_hash_string) do
    if Application.get_env(:explorer, Explorer.SmartContract.RustVerifierInterfaceBehaviour)[:eth_bytecode_db?] do
-      LookUpSmartContractSourcesOnDemand.trigger_fetch(%Address{hash: address_hash_string}, nil)
+      LookUpSmartContractSourcesOnDemand.trigger_fetch(address_hash_string, nil, nil)
    else
      if Application.get_env(:explorer, Explorer.ThirdPartyIntegrations.Sourcify)[:enabled] do
        check_by_address_in_sourcify(