Add new metadata fields and add x-api-key header to eth-bytecode-db r… (#8750)

* Add new metadata fields and add x-api-key header to eth-bytecode-db request

* Add /api/v2/import/smart-contracts/{address_hash} endpoint

CHANGELOG.md

@@ -5,6 +5,7 @@
 ### Features
 
 - [#8795](https://github.com/blockscout/blockscout/pull/8795) - Disable catchup indexer by env
+- [#8750](https://github.com/blockscout/blockscout/pull/8750) - Support new eth-bytecode-db request metadata fields
 
 ### Fixes
 

apps/block_scout_web/lib/block_scout_web/api_router.ex

@@ -181,6 +181,7 @@ defmodule BlockScoutWeb.ApiRouter do
     pipe_through(:api_v2_no_session)
 
     post("/token-info", V2.ImportController, :import_token_info)
+    get("/smart-contracts/:address_hash_param", V2.ImportController, :try_to_search_contract)
   end
 
   scope "/v2", as: :api_v2 do

apps/block_scout_web/lib/block_scout_web/controllers/api/v2/fallback_controller.ex

@@ -130,7 +130,7 @@ defmodule BlockScoutWeb.API.V2.FallbackController do
     |> render(:message, %{message: @restricted_access})
   end
 
-  def call(conn, {:already_verified, true}) do
+  def call(conn, {:already_verified, _}) do
     Logger.error(fn ->
       ["#{@verification_failed}: #{@already_verified}"]
     end)

apps/block_scout_web/lib/block_scout_web/controllers/api/v2/import_controller.ex

@@ -3,7 +3,11 @@ defmodule BlockScoutWeb.API.V2.ImportController do
 
   alias BlockScoutWeb.API.V2.ApiView
   alias Explorer.{Chain, Repo}
-  alias Explorer.Chain.Token
+  alias Explorer.Chain.{Data, Token}
+  alias Explorer.Chain.Fetcher.LookUpSmartContractSourcesOnDemand
+  alias Explorer.SmartContract.EthBytecodeDBInterface
+
+  import Explorer.SmartContract.Helper, only: [prepare_bytecode_for_microservice: 3, contract_creation_input: 1]
 
   require Logger
   @api_true [api?: true]
@@ -48,6 +52,47 @@ defmodule BlockScoutWeb.API.V2.ImportController do
     end
   end
 
+  @doc """
+    Function to handle request at:
+      `/api/v2/smart-contracts/{address_hash_param}`
+
+    Needed to try to import unverified smart contracts via eth-bytecode-db (`/api/v2/bytecodes/sources:search` method).
+    Protected by `x-api-key` header.
+  """
+  @spec try_to_search_contract(Plug.Conn.t(), map()) ::
+          {:already_verified, nil | Explorer.Chain.SmartContract.t()}
+          | {:api_key, nil | binary()}
+          | {:format, :error}
+          | {:not_found, {:error, :not_found}}
+          | {:sensitive_endpoints_api_key, any()}
+          | Plug.Conn.t()
+  def try_to_search_contract(conn, %{"address_hash_param" => address_hash_string}) do
+    with {:sensitive_endpoints_api_key, api_key} when not is_nil(api_key) <-
+           {:sensitive_endpoints_api_key, Application.get_env(:block_scout_web, :sensitive_endpoints_api_key)},
+         {:api_key, ^api_key} <- {:api_key, get_api_key_header(conn)},
+         {:format, {:ok, address_hash}} <- {:format, Chain.string_to_address_hash(address_hash_string)},
+         {:not_found, {:ok, address}} <- {:not_found, Chain.hash_to_address(address_hash, @api_true, false)},
+         {:already_verified, smart_contract} when is_nil(smart_contract) <-
+           {:already_verified, Chain.address_hash_to_smart_contract(address_hash, @api_true)} do
+      creation_tx_input = contract_creation_input(address.hash)
+
+      with {:ok, %{"sourceType" => type} = source} <-
+             %{}
+             |> prepare_bytecode_for_microservice(creation_tx_input, Data.to_string(address.contract_code))
+             |> EthBytecodeDBInterface.search_contract_in_eth_bytecode_internal_db(),
+           {:ok, _} <- LookUpSmartContractSourcesOnDemand.process_contract_source(type, source, address.hash) do
+        conn
+        |> put_view(ApiView)
+        |> render(:message, %{message: "Success"})
+      else
+        _ ->
+          conn
+          |> put_view(ApiView)
+          |> render(:message, %{message: "Contract was not imported"})
+      end
+    end
+  end
+
   defp valid_url?(url) when is_binary(url) do
     uri = URI.parse(url)
     uri.scheme != nil && uri.host =~ "."
@@ -74,4 +119,14 @@ defmodule BlockScoutWeb.API.V2.ImportController do
   end
 
   defp put_token_string_field(changeset, _token_symbol, _field), do: changeset
+
+  defp get_api_key_header(conn) do
+    case get_req_header(conn, "x-api-key") do
+      [api_key] ->
+        api_key
+
+      _ ->
+        nil
+    end
+  end
 end

apps/explorer/lib/explorer/chain/smart_contract.ex

@@ -16,7 +16,7 @@ defmodule Explorer.Chain.SmartContract do
   alias EthereumJSONRPC.Contract
   alias Explorer.Counters.AverageBlockTime
   alias Explorer.{Chain, Repo}
-  alias Explorer.Chain.{Address, ContractMethod, DecompiledSmartContract, Hash}
+  alias Explorer.Chain.{Address, ContractMethod, Data, DecompiledSmartContract, Hash, InternalTransaction, Transaction}
   alias Explorer.Chain.SmartContract.ExternalLibrary
   alias Explorer.SmartContract.Reader
   alias Timex.Duration
@@ -965,4 +965,48 @@ defmodule Explorer.Chain.SmartContract do
 
     Chain.select_repo(options).one(query)
   end
+
+  @doc """
+    Extracts creation bytecode (`init`) and transaction (`tx`) or
+      internal transaction (`internal_tx`) where the contract was created.
+  """
+  @spec creation_tx_with_bytecode(binary() | Hash.t()) ::
+          %{init: binary(), tx: Transaction.t()} | %{init: binary(), internal_tx: InternalTransaction.t()} | nil
+  def creation_tx_with_bytecode(address_hash) do
+    creation_tx_query =
+      from(
+        tx in Transaction,
+        where: tx.created_contract_address_hash == ^address_hash,
+        where: tx.status == ^1
+      )
+
+    tx =
+      creation_tx_query
+      |> Repo.one()
+
+    if tx do
+      with %{input: input} <- tx do
+        %{init: Data.to_string(input), tx: tx}
+      end
+    else
+      creation_int_tx_query =
+        from(
+          itx in InternalTransaction,
+          join: t in assoc(itx, :transaction),
+          where: itx.created_contract_address_hash == ^address_hash,
+          where: t.status == ^1
+        )
+
+      internal_tx = creation_int_tx_query |> Repo.one()
+
+      case internal_tx do
+        %{init: init} ->
+          init_str = Data.to_string(init)
+          %{init: init_str, internal_tx: internal_tx}
+
+        _ ->
+          nil
+      end
+    end
+  end
 end

apps/explorer/lib/explorer/smart_contract/eth_bytecode_db_interface.ex

@@ -17,6 +17,15 @@ defmodule Explorer.SmartContract.EthBytecodeDBInterface do
     end
   end
 
+  @doc """
+    Function to search smart contracts in eth-bytecode-db, similar to `search_contract/2` but
+      this function uses only `/api/v2/bytecodes/sources:search` method
+  """
+  @spec search_contract_in_eth_bytecode_internal_db(map()) :: {:error, any} | {:ok, any}
+  def search_contract_in_eth_bytecode_internal_db(%{"bytecode" => _, "bytecodeType" => _} = body) do
+    http_post_request(bytecode_search_sources_url(), body)
+  end
+
   def process_verifier_response(%{"sourcifySources" => [src | _]}) do
     {:ok, Map.put(src, "sourcify?", true)}
   end

apps/explorer/lib/explorer/smart_contract/helper.ex

@@ -4,6 +4,7 @@ defmodule Explorer.SmartContract.Helper do
   """
 
   alias Explorer.Chain
+  alias Explorer.Chain.{Hash, SmartContract}
   alias Phoenix.HTML
 
   def queriable_method?(method) do
@@ -135,4 +136,61 @@ defmodule Explorer.SmartContract.Helper do
         nil
     end
   end
+
+  @doc """
+    Returns a tuple: `{creation_bytecode, deployed_bytecode, metadata}` where `metadata` is a map:
+      {
+        "blockNumber": "string",
+        "chainId": "string",
+        "contractAddress": "string",
+        "creationCode": "string",
+        "deployer": "string",
+        "runtimeCode": "string",
+        "transactionHash": "string",
+        "transactionIndex": "string"
+      }
+
+    Metadata will be sent to a verifier microservice
+  """
+  @spec fetch_data_for_verification(binary() | Hash.t()) :: {binary() | nil, binary(), map()}
+  def fetch_data_for_verification(address_hash) do
+    deployed_bytecode = Chain.smart_contract_bytecode(address_hash)
+
+    metadata = %{
+      "contractAddress" => to_string(address_hash),
+      "runtimeCode" => to_string(deployed_bytecode),
+      "chainId" => Application.get_env(:block_scout_web, :chain_id)
+    }
+
+    case SmartContract.creation_tx_with_bytecode(address_hash) do
+      %{init: init, tx: tx} ->
+        {init, deployed_bytecode, tx |> tx_to_metadata(init) |> Map.merge(metadata)}
+
+      %{init: init, internal_tx: internal_tx} ->
+        {init, deployed_bytecode, internal_tx |> internal_tx_to_metadata(init) |> Map.merge(metadata)}
+
+      _ ->
+        {nil, deployed_bytecode, metadata}
+    end
+  end
+
+  defp tx_to_metadata(tx, init) do
+    %{
+      "blockNumber" => to_string(tx.block_number),
+      "transactionHash" => to_string(tx.hash),
+      "transactionIndex" => to_string(tx.index),
+      "deployer" => to_string(tx.from_address_hash),
+      "creationCode" => to_string(init)
+    }
+  end
+
+  defp internal_tx_to_metadata(internal_tx, init) do
+    %{
+      "blockNumber" => to_string(internal_tx.block_number),
+      "transactionHash" => to_string(internal_tx.transaction_hash),
+      "transactionIndex" => to_string(internal_tx.transaction_index),
+      "deployer" => to_string(internal_tx.from_address_hash),
+      "creationCode" => to_string(init)
+    }
+  end
 end

apps/explorer/lib/explorer/smart_contract/rust_verifier_interface_behaviour.ex

@@ -22,9 +22,9 @@ defmodule Explorer.SmartContract.RustVerifierInterfaceBehaviour do
               "optimizationRuns" => _,
               "libraries" => _
             } = body,
-            address_hash
+            metadata
           ) do
-        http_post_request(solidity_multiple_files_verification_url(), append_metadata(body, address_hash))
+        http_post_request(solidity_multiple_files_verification_url(), append_metadata(body, metadata), true)
       end
 
       def verify_standard_json_input(
@@ -34,9 +34,9 @@ defmodule Explorer.SmartContract.RustVerifierInterfaceBehaviour do
               "compilerVersion" => _,
               "input" => _
             } = body,
-            address_hash
+            metadata
           ) do
-        http_post_request(solidity_standard_json_verification_url(), append_metadata(body, address_hash))
+        http_post_request(solidity_standard_json_verification_url(), append_metadata(body, metadata), true)
       end
 
       def vyper_verify_multipart(
@@ -46,9 +46,9 @@ defmodule Explorer.SmartContract.RustVerifierInterfaceBehaviour do
               "compilerVersion" => _,
               "sourceFiles" => _
             } = body,
-            address_hash
+            metadata
           ) do
-        http_post_request(vyper_multiple_files_verification_url(), append_metadata(body, address_hash))
+        http_post_request(vyper_multiple_files_verification_url(), append_metadata(body, metadata), true)
       end
 
       def vyper_verify_standard_json(
@@ -58,15 +58,17 @@ defmodule Explorer.SmartContract.RustVerifierInterfaceBehaviour do
               "compilerVersion" => _,
               "input" => _
             } = body,
-            address_hash
+            metadata
           ) do
-        http_post_request(vyper_standard_json_verification_url(), append_metadata(body, address_hash))
+        http_post_request(vyper_standard_json_verification_url(), append_metadata(body, metadata), true)
       end
 
-      def http_post_request(url, body) do
+      def http_post_request(url, body, is_verification_request? \\ false) do
         headers = [{"Content-Type", "application/json"}]
 
-        case HTTPoison.post(url, Jason.encode!(body), headers, recv_timeout: @post_timeout) do
+        case HTTPoison.post(url, Jason.encode!(body), maybe_put_api_key_header(headers, is_verification_request?),
+               recv_timeout: @post_timeout
+             ) do
           {:ok, %Response{body: body, status_code: _}} ->
             process_verifier_response(body)
 
@@ -86,6 +88,18 @@ defmodule Explorer.SmartContract.RustVerifierInterfaceBehaviour do
         end
       end
 
+      defp maybe_put_api_key_header(headers, false), do: headers
+
+      defp maybe_put_api_key_header(headers, true) do
+        api_key = Application.get_env(:explorer, Explorer.SmartContract.RustVerifierInterfaceBehaviour)[:api_key]
+
+        if api_key do
+          [{"x-api-key", api_key} | headers]
+        else
+          headers
+        end
+      end
+
       def http_get_request(url) do
         case HTTPoison.get(url) do
           {:ok, %Response{body: body, status_code: 200}} ->
@@ -163,12 +177,9 @@ defmodule Explorer.SmartContract.RustVerifierInterfaceBehaviour do
 
       def enabled?, do: Application.get_env(:explorer, Explorer.SmartContract.RustVerifierInterfaceBehaviour)[:enabled]
 
-      defp append_metadata(body, address_hash) when is_map(body) do
+      defp append_metadata(body, metadata) when is_map(body) do
         body
-        |> Map.put("metadata", %{
-          "chainId" => Application.get_env(:block_scout_web, :chain_id),
-          "contractAddress" => to_string(address_hash)
-        })
+        |> Map.put("metadata", metadata)
       end
     end
   end

apps/explorer/lib/explorer/smart_contract/solidity/verifier.ex

@@ -9,9 +9,12 @@ defmodule Explorer.SmartContract.Solidity.Verifier do
   """
 
   import Explorer.SmartContract.Helper,
-    only: [cast_libraries: 1, prepare_bytecode_for_microservice: 3, contract_creation_input: 1]
+    only: [
+      cast_libraries: 1,
+      fetch_data_for_verification: 1,
+      prepare_bytecode_for_microservice: 3
+    ]
 
-  # import  Explorer.Chain.SmartContract, only: [:function_description]
   alias ABI.{FunctionSelector, TypeDecoder}
   alias Explorer.Chain
   alias Explorer.Chain.{Data, Hash, SmartContract}
@@ -40,9 +43,7 @@ defmodule Explorer.SmartContract.Solidity.Verifier do
   end
 
   defp evaluate_authenticity_inner(true, address_hash, params) do
-    deployed_bytecode = Chain.smart_contract_bytecode(address_hash)
-
-    creation_tx_input = contract_creation_input(address_hash)
+    {creation_tx_input, deployed_bytecode, verifier_metadata} = fetch_data_for_verification(address_hash)
 
     %{}
     |> prepare_bytecode_for_microservice(creation_tx_input, deployed_bytecode)
@@ -54,7 +55,7 @@ defmodule Explorer.SmartContract.Solidity.Verifier do
     |> Map.put("optimizationRuns", prepare_optimization_runs(params["optimization"], params["optimization_runs"]))
     |> Map.put("evmVersion", Map.get(params, "evm_version", "default"))
     |> Map.put("compilerVersion", params["compiler_version"])
-    |> RustVerifierInterface.verify_multi_part(address_hash)
+    |> RustVerifierInterface.verify_multi_part(verifier_metadata)
   end
 
   defp evaluate_authenticity_inner(false, address_hash, params) do
@@ -124,14 +125,12 @@ defmodule Explorer.SmartContract.Solidity.Verifier do
   end
 
   def evaluate_authenticity_via_standard_json_input_inner(true, address_hash, params, json_input) do
-    deployed_bytecode = Chain.smart_contract_bytecode(address_hash)
-
-    creation_tx_input = contract_creation_input(address_hash)
+    {creation_tx_input, deployed_bytecode, verifier_metadata} = fetch_data_for_verification(address_hash)
 
     %{"compilerVersion" => params["compiler_version"]}
     |> prepare_bytecode_for_microservice(creation_tx_input, deployed_bytecode)
     |> Map.put("input", json_input)
-    |> RustVerifierInterface.verify_standard_json_input(address_hash)
+    |> RustVerifierInterface.verify_standard_json_input(verifier_metadata)
   end
 
   def evaluate_authenticity_via_standard_json_input_inner(false, address_hash, params, json_input) do
@@ -139,9 +138,7 @@ defmodule Explorer.SmartContract.Solidity.Verifier do
   end
 
   def evaluate_authenticity_via_multi_part_files(address_hash, params, files) do
-    deployed_bytecode = Chain.smart_contract_bytecode(address_hash)
-
-    creation_tx_input = contract_creation_input(address_hash)
+    {creation_tx_input, deployed_bytecode, verifier_metadata} = fetch_data_for_verification(address_hash)
 
     %{}
     |> prepare_bytecode_for_microservice(creation_tx_input, deployed_bytecode)
@@ -150,7 +147,7 @@ defmodule Explorer.SmartContract.Solidity.Verifier do
     |> Map.put("optimizationRuns", prepare_optimization_runs(params["optimization"], params["optimization_runs"]))
     |> Map.put("evmVersion", Map.get(params, "evm_version", "default"))
     |> Map.put("compilerVersion", params["compiler_version"])
-    |> RustVerifierInterface.verify_multi_part(address_hash)
+    |> RustVerifierInterface.verify_multi_part(verifier_metadata)
   end
 
   defp verify(address_hash, params, json_input) do

apps/explorer/lib/explorer/smart_contract/vyper/verifier.ex

@@ -9,10 +9,11 @@ defmodule Explorer.SmartContract.Vyper.Verifier do
   """
   require Logger
 
-  alias Explorer.Chain
   alias Explorer.SmartContract.Vyper.CodeCompiler
   alias Explorer.SmartContract.RustVerifierInterface
-  import Explorer.SmartContract.Helper, only: [prepare_bytecode_for_microservice: 3, contract_creation_input: 1]
+
+  import Explorer.SmartContract.Helper,
+    only: [fetch_data_for_verification: 1, prepare_bytecode_for_microservice: 3, contract_creation_input: 1]
 
   def evaluate_authenticity(_, %{"contract_source_code" => ""}),
     do: {:error, :contract_source_code}
@@ -34,7 +35,7 @@ defmodule Explorer.SmartContract.Vyper.Verifier do
   def evaluate_authenticity(address_hash, params, files) do
     try do
       if RustVerifierInterface.enabled?() do
-        vyper_verify_multipart(params, fetch_bytecode(address_hash), params["evm_version"], files, address_hash)
+        vyper_verify_multipart(params, params["evm_version"], files, address_hash)
       end
     rescue
       exception ->
@@ -50,7 +51,7 @@ defmodule Explorer.SmartContract.Vyper.Verifier do
   def evaluate_authenticity_standard_json(%{"address_hash" => address_hash} = params) do
     try do
       if RustVerifierInterface.enabled?() do
-        vyper_verify_standard_json(params, fetch_bytecode(address_hash), address_hash)
+        vyper_verify_standard_json(params, address_hash)
       end
     rescue
       exception ->
@@ -66,7 +67,6 @@ defmodule Explorer.SmartContract.Vyper.Verifier do
   defp evaluate_authenticity_inner(true, address_hash, params) do
     vyper_verify_multipart(
       params,
-      fetch_bytecode(address_hash),
       params["evm_version"],
       %{
         "#{params["name"]}.vy" => params["contract_source_code"]
@@ -79,13 +79,6 @@ defmodule Explorer.SmartContract.Vyper.Verifier do
     verify(address_hash, params)
   end
 
-  def fetch_bytecode(address_hash) do
-    deployed_bytecode = Chain.smart_contract_bytecode(address_hash)
-    creation_tx_input = contract_creation_input(address_hash)
-
-    prepare_bytecode_for_microservice(%{}, creation_tx_input, deployed_bytecode)
-  end
-
   defp verify(address_hash, params) do
     contract_source_code = Map.fetch!(params, "contract_source_code")
     compiler_version = Map.fetch!(params, "compiler_version")
@@ -123,19 +116,25 @@ defmodule Explorer.SmartContract.Vyper.Verifier do
     end
   end
 
-  defp vyper_verify_multipart(params, bytecode_map, evm_version, files, address_hash) do
-    bytecode_map
+  defp vyper_verify_multipart(params, evm_version, files, address_hash) do
+    {creation_tx_input, deployed_bytecode, verifier_metadata} = fetch_data_for_verification(address_hash)
+
+    %{}
+    |> prepare_bytecode_for_microservice(creation_tx_input, deployed_bytecode)
     |> Map.put("evmVersion", evm_version)
     |> Map.put("sourceFiles", files)
     |> Map.put("compilerVersion", params["compiler_version"])
     |> Map.put("interfaces", params["interfaces"] || %{})
-    |> RustVerifierInterface.vyper_verify_multipart(address_hash)
+    |> RustVerifierInterface.vyper_verify_multipart(verifier_metadata)
   end
 
-  defp vyper_verify_standard_json(params, bytecode_map, address_hash) do
-    bytecode_map
+  defp vyper_verify_standard_json(params, address_hash) do
+    {creation_tx_input, deployed_bytecode, verifier_metadata} = fetch_data_for_verification(address_hash)
+
+    %{}
+    |> prepare_bytecode_for_microservice(creation_tx_input, deployed_bytecode)
     |> Map.put("compilerVersion", params["compiler_version"])
     |> Map.put("input", params["input"])
-    |> RustVerifierInterface.vyper_verify_standard_json(address_hash)
+    |> RustVerifierInterface.vyper_verify_standard_json(verifier_metadata)
   end
 end

config/runtime.exs

@@ -379,7 +379,8 @@ config :explorer, Explorer.SmartContract.RustVerifierInterfaceBehaviour,
   service_url: System.get_env("MICROSERVICE_SC_VERIFIER_URL") || "https://eth-bytecode-db.services.blockscout.com/",
   enabled: enabled?,
   type: type,
-  eth_bytecode_db?: enabled? && type == "eth_bytecode_db"
+  eth_bytecode_db?: enabled? && type == "eth_bytecode_db",
+  api_key: System.get_env("MICROSERVICE_SC_VERIFIER_API_KEY")
 
 config :explorer, Explorer.Visualize.Sol2uml,
   service_url: System.get_env("MICROSERVICE_VISUALIZE_SOL2UML_URL"),