Integrate /api/v2/bytecodes/sources:search-all of eth_bytecode_db

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### Features
 
+- [#8472](https://github.com/blockscout/blockscout/pull/8472) - Integrate `/api/v2/bytecodes/sources:search-all` of `eth_bytecode_db`
 - [#8544](https://github.com/blockscout/blockscout/pull/8544) - Fix `nil` `"structLogs"`
 - [#8561](https://github.com/blockscout/blockscout/pull/8561), [#8564](https://github.com/blockscout/blockscout/pull/8564) - Get historical market cap data from CoinGecko
 - [#8386](https://github.com/blockscout/blockscout/pull/8386) - Add `owner_address_hash` to the `token_instances`

apps/block_scout_web/lib/block_scout_web/controllers/api/v2/smart_contract_controller.ex

@@ -14,7 +14,6 @@ defmodule BlockScoutWeb.API.V2.SmartContractController do
   alias Explorer.Chain
   alias Explorer.Chain.SmartContract
   alias Explorer.SmartContract.{Reader, Writer}
-  alias Explorer.SmartContract.Solidity.PublishHelper
 
   @smart_contract_address_options [
     necessity_by_association: %{
@@ -32,9 +31,8 @@ defmodule BlockScoutWeb.API.V2.SmartContractController do
   def smart_contract(conn, %{"address_hash" => address_hash_string} = params) do
     with {:format, {:ok, address_hash}} <- {:format, Chain.string_to_address_hash(address_hash_string)},
          {:ok, false} <- AccessHelper.restricted_access?(address_hash_string, params),
-         _ <- PublishHelper.check_and_verify(address_hash_string),
          {:not_found, {:ok, address}} <-
-           {:not_found, Chain.find_contract_address(address_hash, @smart_contract_address_options, true)} do
+           {:not_found, Chain.find_contract_address(address_hash, @smart_contract_address_options, false)} do
       conn
       |> put_status(200)
       |> render(:smart_contract, %{address: address})

apps/block_scout_web/test/block_scout_web/controllers/api/v2/smart_contract_controller_test.exs

@@ -303,9 +303,14 @@ defmodule BlockScoutWeb.API.V2.SmartContractControllerTest do
 
       assert correct_response == response
     end
+  end
 
-    test "automatically verify contract via Eth Bytecode Interface", %{conn: conn} do
+  describe "/smart-contracts/{address_hash} <> eth_bytecode_db" do
+    test "automatically verify contract", %{conn: conn} do
       {:ok, pid} = Explorer.Chain.Fetcher.LookUpSmartContractSourcesOnDemand.start_link([])
+      old_chain_id = Application.get_env(:block_scout_web, :chain_id)
+
+      Application.put_env(:block_scout_web, :chain_id, 5)
 
       bypass = Bypass.open()
       eth_bytecode_response = File.read!("./test/support/fixture/smart_contract/eth_bytecode_db_search_response.json")
@@ -333,7 +338,77 @@ defmodule BlockScoutWeb.API.V2.SmartContractControllerTest do
         |> socket("no_id", %{})
         |> subscribe_and_join(topic)
 
-      Bypass.expect_once(bypass, "POST", "/api/v2/bytecodes/sources:search", fn conn ->
+      Bypass.expect_once(bypass, "POST", "/api/v2/bytecodes/sources_search_all", fn conn ->
+        Conn.resp(conn, 200, eth_bytecode_response)
+      end)
+
+      request = get(conn, "/api/v2/smart-contracts/#{Address.checksum(address.hash)}")
+
+      assert_receive %Phoenix.Socket.Message{
+                       payload: %{},
+                       event: "smart_contract_was_verified",
+                       topic: ^topic
+                     },
+                     :timer.seconds(1)
+
+      response = json_response(request, 200)
+
+      assert response ==
+               %{
+                 "is_self_destructed" => false,
+                 "deployed_bytecode" => to_string(address.contract_code),
+                 "creation_bytecode" =>
+                   "0x608060405234801561001057600080fd5b5060df8061001f6000396000f3006080604052600436106049576000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff16806360fe47b114604e5780636d4ce63c146078575b600080fd5b348015605957600080fd5b5060766004803603810190808035906020019092919050505060a0565b005b348015608357600080fd5b50608a60aa565b6040518082815260200191505060405180910390f35b8060008190555050565b600080549050905600a165627a7a7230582061b7676067d537e410bb704932a9984739a959416170ea17bda192ac1218d2790029"
+               }
+
+      request = get(conn, "/api/v2/smart-contracts/#{Address.checksum(address.hash)}")
+      assert response = json_response(request, 200)
+      assert %{"is_verified" => true} = response
+      assert %{"is_verified_via_eth_bytecode_db" => true} = response
+      assert %{"is_partially_verified" => true} = response
+      assert %{"is_fully_verified" => false} = response
+
+      Application.put_env(:block_scout_web, :chain_id, old_chain_id)
+      Application.put_env(:explorer, Explorer.SmartContract.RustVerifierInterfaceBehaviour, old_env)
+      Bypass.down(bypass)
+      GenServer.stop(pid)
+    end
+
+    test "automatically verify contract using search-all (ethBytecodeDbSources) endpoint", %{conn: conn} do
+      {:ok, pid} = Explorer.Chain.Fetcher.LookUpSmartContractSourcesOnDemand.start_link([])
+      old_chain_id = Application.get_env(:block_scout_web, :chain_id)
+
+      Application.put_env(:block_scout_web, :chain_id, 5)
+
+      bypass = Bypass.open()
+
+      eth_bytecode_response =
+        File.read!("./test/support/fixture/smart_contract/eth_bytecode_db_search_all_local_sources_response.json")
+
+      old_env = Application.get_env(:explorer, Explorer.SmartContract.RustVerifierInterfaceBehaviour)
+
+      Application.put_env(:explorer, Explorer.SmartContract.RustVerifierInterfaceBehaviour,
+        service_url: "http://localhost:#{bypass.port}",
+        enabled: true
+      )
+
+      address = insert(:contract_address)
+
+      insert(:transaction,
+        created_contract_address_hash: address.hash,
+        input:
+          "0x608060405234801561001057600080fd5b5060df8061001f6000396000f3006080604052600436106049576000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff16806360fe47b114604e5780636d4ce63c146078575b600080fd5b348015605957600080fd5b5060766004803603810190808035906020019092919050505060a0565b005b348015608357600080fd5b50608a60aa565b6040518082815260200191505060405180910390f35b8060008190555050565b600080549050905600a165627a7a7230582061b7676067d537e410bb704932a9984739a959416170ea17bda192ac1218d2790029"
+      )
+      |> with_block()
+
+      topic = "addresses:#{address.hash}"
+
+      {:ok, _reply, _socket} =
+        BlockScoutWeb.UserSocketV2
+        |> socket("no_id", %{})
+        |> subscribe_and_join(topic)
+
+      Bypass.expect_once(bypass, "POST", "/api/v2/bytecodes/sources_search_all", fn conn ->
         Conn.resp(conn, 200, eth_bytecode_response)
       end)
 
@@ -363,13 +438,236 @@ defmodule BlockScoutWeb.API.V2.SmartContractControllerTest do
       assert %{"is_partially_verified" => true} = response
       assert %{"is_fully_verified" => false} = response
 
+      smart_contract = Jason.decode!(eth_bytecode_response)["ethBytecodeDbSources"] |> List.first()
+      assert response["compiler_settings"] == Jason.decode!(smart_contract["compilerSettings"])
+      assert response["name"] == smart_contract["contractName"]
+      assert response["compiler_version"] == smart_contract["compilerVersion"]
+      assert response["file_path"] == smart_contract["fileName"]
+      assert response["constructor_args"] == smart_contract["constructorArguments"]
+      assert response["abi"] == Jason.decode!(smart_contract["abi"])
+
+      assert response["decoded_constructor_args"] == [
+               [
+                 "0xc35dadb65012ec5796536bd9864ed8773abc74c4",
+                 %{
+                   "internalType" => "address",
+                   "name" => "_factory",
+                   "type" => "address"
+                 }
+               ],
+               [
+                 "0xb4fbf271143f4fbf7b91a5ded31805e42b2208d6",
+                 %{
+                   "internalType" => "address",
+                   "name" => "_WETH",
+                   "type" => "address"
+                 }
+               ]
+             ]
+
+      assert response["source_code"] == smart_contract["sourceFiles"][smart_contract["fileName"]]
+
+      assert response["external_libraries"] == [
+               %{
+                 "address_hash" => "0x00000000D41867734BBee4C6863D9255b2b06aC1",
+                 "name" => "__CACHE_BREAKER__"
+               }
+             ]
+
+      additional_sources =
+        for file_name <- Map.keys(smart_contract["sourceFiles"]), smart_contract["fileName"] != file_name do
+          %{
+            "source_code" => smart_contract["sourceFiles"][file_name],
+            "file_path" => file_name
+          }
+        end
+
+      assert response["additional_sources"] |> Enum.sort_by(fn x -> x["file_path"] end) ==
+               additional_sources |> Enum.sort_by(fn x -> x["file_path"] end)
+
+      Application.put_env(:block_scout_web, :chain_id, old_chain_id)
       Application.put_env(:explorer, Explorer.SmartContract.RustVerifierInterfaceBehaviour, old_env)
       Bypass.down(bypass)
       GenServer.stop(pid)
     end
 
-    test "check fetch interval for LookUpSmartContractSourcesOnDemand", %{conn: conn} do
+    test "automatically verify contract using search-all (sourcifySources) endpoint", %{conn: conn} do
       {:ok, pid} = Explorer.Chain.Fetcher.LookUpSmartContractSourcesOnDemand.start_link([])
+      old_chain_id = Application.get_env(:block_scout_web, :chain_id)
+
+      Application.put_env(:block_scout_web, :chain_id, 5)
+
+      bypass = Bypass.open()
+
+      eth_bytecode_response =
+        File.read!("./test/support/fixture/smart_contract/eth_bytecode_db_search_all_sourcify_sources_response.json")
+
+      old_env = Application.get_env(:explorer, Explorer.SmartContract.RustVerifierInterfaceBehaviour)
+
+      Application.put_env(:explorer, Explorer.SmartContract.RustVerifierInterfaceBehaviour,
+        service_url: "http://localhost:#{bypass.port}",
+        enabled: true
+      )
+
+      address = insert(:contract_address)
+
+      insert(:transaction,
+        created_contract_address_hash: address.hash,
+        input:
+          "0x608060405234801561001057600080fd5b5060df8061001f6000396000f3006080604052600436106049576000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff16806360fe47b114604e5780636d4ce63c146078575b600080fd5b348015605957600080fd5b5060766004803603810190808035906020019092919050505060a0565b005b348015608357600080fd5b50608a60aa565b6040518082815260200191505060405180910390f35b8060008190555050565b600080549050905600a165627a7a7230582061b7676067d537e410bb704932a9984739a959416170ea17bda192ac1218d2790029"
+      )
+      |> with_block()
+
+      topic = "addresses:#{address.hash}"
+
+      {:ok, _reply, _socket} =
+        BlockScoutWeb.UserSocketV2
+        |> socket("no_id", %{})
+        |> subscribe_and_join(topic)
+
+      Bypass.expect_once(bypass, "POST", "/api/v2/bytecodes/sources_search_all", fn conn ->
+        Conn.resp(conn, 200, eth_bytecode_response)
+      end)
+
+      request = get(conn, "/api/v2/smart-contracts/#{Address.checksum(address.hash)}")
+
+      assert_receive %Phoenix.Socket.Message{
+                       payload: %{},
+                       event: "smart_contract_was_verified",
+                       topic: ^topic
+                     },
+                     :timer.seconds(1)
+
+      response = json_response(request, 200)
+
+      assert response ==
+               %{
+                 "is_self_destructed" => false,
+                 "deployed_bytecode" => to_string(address.contract_code),
+                 "creation_bytecode" =>
+                   "0x608060405234801561001057600080fd5b5060df8061001f6000396000f3006080604052600436106049576000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff16806360fe47b114604e5780636d4ce63c146078575b600080fd5b348015605957600080fd5b5060766004803603810190808035906020019092919050505060a0565b005b348015608357600080fd5b50608a60aa565b6040518082815260200191505060405180910390f35b8060008190555050565b600080549050905600a165627a7a7230582061b7676067d537e410bb704932a9984739a959416170ea17bda192ac1218d2790029"
+               }
+
+      request = get(conn, "/api/v2/smart-contracts/#{Address.checksum(address.hash)}")
+      assert response = json_response(request, 200)
+      assert %{"is_verified" => true} = response
+      assert %{"is_verified_via_eth_bytecode_db" => true} = response
+      assert %{"is_verified_via_sourcify" => true} = response
+      assert %{"is_partially_verified" => true} = response
+      assert %{"is_fully_verified" => false} = response
+      assert response["file_path"] == "Test.sol"
+
+      Application.put_env(:block_scout_web, :chain_id, old_chain_id)
+      Application.put_env(:explorer, Explorer.SmartContract.RustVerifierInterfaceBehaviour, old_env)
+      Bypass.down(bypass)
+      GenServer.stop(pid)
+    end
+
+    test "automatically verify contract using search-all (sourcifySources with libraries) endpoint", %{conn: conn} do
+      {:ok, pid} = Explorer.Chain.Fetcher.LookUpSmartContractSourcesOnDemand.start_link([])
+      old_chain_id = Application.get_env(:block_scout_web, :chain_id)
+
+      Application.put_env(:block_scout_web, :chain_id, 5)
+
+      bypass = Bypass.open()
+
+      eth_bytecode_response =
+        File.read!(
+          "./test/support/fixture/smart_contract/eth_bytecode_db_search_all_sourcify_sources_with_libs_response.json"
+        )
+
+      old_env = Application.get_env(:explorer, Explorer.SmartContract.RustVerifierInterfaceBehaviour)
+
+      Application.put_env(:explorer, Explorer.SmartContract.RustVerifierInterfaceBehaviour,
+        service_url: "http://localhost:#{bypass.port}",
+        enabled: true
+      )
+
+      address = insert(:contract_address)
+
+      insert(:transaction,
+        created_contract_address_hash: address.hash,
+        input:
+          "0x608060405234801561001057600080fd5b5060df8061001f6000396000f3006080604052600436106049576000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff16806360fe47b114604e5780636d4ce63c146078575b600080fd5b348015605957600080fd5b5060766004803603810190808035906020019092919050505060a0565b005b348015608357600080fd5b50608a60aa565b6040518082815260200191505060405180910390f35b8060008190555050565b600080549050905600a165627a7a7230582061b7676067d537e410bb704932a9984739a959416170ea17bda192ac1218d2790029"
+      )
+      |> with_block()
+
+      topic = "addresses:#{address.hash}"
+
+      {:ok, _reply, _socket} =
+        BlockScoutWeb.UserSocketV2
+        |> socket("no_id", %{})
+        |> subscribe_and_join(topic)
+
+      Bypass.expect_once(bypass, "POST", "/api/v2/bytecodes/sources_search_all", fn conn ->
+        Conn.resp(conn, 200, eth_bytecode_response)
+      end)
+
+      request = get(conn, "/api/v2/smart-contracts/#{Address.checksum(address.hash)}")
+
+      assert_receive %Phoenix.Socket.Message{
+                       payload: %{},
+                       event: "smart_contract_was_verified",
+                       topic: ^topic
+                     },
+                     :timer.seconds(1)
+
+      response = json_response(request, 200)
+
+      assert response ==
+               %{
+                 "is_self_destructed" => false,
+                 "deployed_bytecode" => to_string(address.contract_code),
+                 "creation_bytecode" =>
+                   "0x608060405234801561001057600080fd5b5060df8061001f6000396000f3006080604052600436106049576000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff16806360fe47b114604e5780636d4ce63c146078575b600080fd5b348015605957600080fd5b5060766004803603810190808035906020019092919050505060a0565b005b348015608357600080fd5b50608a60aa565b6040518082815260200191505060405180910390f35b8060008190555050565b600080549050905600a165627a7a7230582061b7676067d537e410bb704932a9984739a959416170ea17bda192ac1218d2790029"
+               }
+
+      request = get(conn, "/api/v2/smart-contracts/#{Address.checksum(address.hash)}")
+      assert response = json_response(request, 200)
+
+      smart_contract = Jason.decode!(eth_bytecode_response)["sourcifySources"] |> List.first()
+
+      assert %{"is_verified" => true} = response
+      assert %{"is_verified_via_eth_bytecode_db" => true} = response
+      assert %{"is_verified_via_sourcify" => true} = response
+      assert %{"is_partially_verified" => true} = response
+      assert %{"is_fully_verified" => false} = response
+      assert response["file_path"] == "src/zkbob/ZkBobPool.sol"
+
+      assert response["external_libraries"] == [
+               %{
+                 "address_hash" => "0x22DE6B06544Ee5Cd907813a04bcdEd149A2f49D2",
+                 "name" => "lib/base58-solidity/contracts/Base58.sol:Base58"
+               },
+               %{
+                 "address_hash" => "0x019d3788F00a7087234f3844CB1ceCe1F9982B7A",
+                 "name" => "src/libraries/ZkAddress.sol:ZkAddress"
+               }
+             ]
+
+      additional_sources =
+        for file_name <- Map.keys(smart_contract["sourceFiles"]), smart_contract["fileName"] != file_name do
+          %{
+            "source_code" => smart_contract["sourceFiles"][file_name],
+            "file_path" => file_name
+          }
+        end
+
+      assert response["additional_sources"] |> Enum.sort_by(fn x -> x["file_path"] end) ==
+               additional_sources |> Enum.sort_by(fn x -> x["file_path"] end)
+
+      Application.put_env(:block_scout_web, :chain_id, old_chain_id)
+      Application.put_env(:explorer, Explorer.SmartContract.RustVerifierInterfaceBehaviour, old_env)
+      Bypass.down(bypass)
+      GenServer.stop(pid)
+    end
+
+    test "check fetch interval for LookUpSmartContractSourcesOnDemand and use sources:search endpoint since chain_id is unset",
+         %{conn: conn} do
+      {:ok, pid} = Explorer.Chain.Fetcher.LookUpSmartContractSourcesOnDemand.start_link([])
+      old_chain_id = Application.get_env(:block_scout_web, :chain_id)
+
+      Application.put_env(:block_scout_web, :chain_id, nil)
 
       bypass = Bypass.open()
       address = insert(:contract_address)
@@ -392,7 +690,7 @@ defmodule BlockScoutWeb.API.V2.SmartContractControllerTest do
 
       Application.put_env(:explorer, Explorer.Chain.Fetcher.LookUpSmartContractSourcesOnDemand, fetch_interval: 0)
 
-      Bypass.expect_once(bypass, "POST", "/api/v2/bytecodes/sources:search", fn conn ->
+      Bypass.expect_once(bypass, "POST", "/api/v2/bytecodes/sources_search", fn conn ->
         Conn.resp(conn, 200, "{\"sources\": []}")
       end)
 
@@ -400,7 +698,7 @@ defmodule BlockScoutWeb.API.V2.SmartContractControllerTest do
 
       :timer.sleep(10)
 
-      Bypass.expect_once(bypass, "POST", "/api/v2/bytecodes/sources:search", fn conn ->
+      Bypass.expect_once(bypass, "POST", "/api/v2/bytecodes/sources_search", fn conn ->
         Conn.resp(conn, 200, "{\"sources\": []}")
       end)
 
@@ -408,7 +706,7 @@ defmodule BlockScoutWeb.API.V2.SmartContractControllerTest do
 
       :timer.sleep(10)
 
-      Bypass.expect_once(bypass, "POST", "/api/v2/bytecodes/sources:search", fn conn ->
+      Bypass.expect_once(bypass, "POST", "/api/v2/bytecodes/sources_search", fn conn ->
         Conn.resp(conn, 200, "{\"sources\": []}")
       end)
 
@@ -420,6 +718,7 @@ defmodule BlockScoutWeb.API.V2.SmartContractControllerTest do
 
       _request = get(conn, "/api/v2/smart-contracts/#{Address.checksum(address.hash)}")
 
+      Application.put_env(:block_scout_web, :chain_id, old_chain_id)
       Application.put_env(:explorer, Explorer.Chain.Fetcher.LookUpSmartContractSourcesOnDemand, old_interval_env)
       Application.put_env(:explorer, Explorer.SmartContract.RustVerifierInterfaceBehaviour, old_env)
       Bypass.down(bypass)

apps/block_scout_web/test/support/fixture/smart_contract/eth_bytecode_db_search_all_sourcify_sources_response.json

@@ -0,0 +1,30 @@
+{
+    "ethBytecodeDbSources": [
+        {
+            "fileName": "Test_eth.sol",
+            "contractName": "Test",
+            "compilerVersion": "v0.8.17+commit.8df45f5f",
+            "compilerSettings": "{\"libraries\":{\"Test.sol\":{}},\"metadata\":{\"bytecodeHash\":\"ipfs\"},\"optimizer\":{\"enabled\":true,\"runs\":199},\"outputSelection\":{\"*\":{\"\":[\"ast\"],\"*\":[\"abi\",\"evm.bytecode\",\"evm.deployedBytecode\",\"evm.methodIdentifiers\"]}}}",
+            "sourceType": "SOLIDITY",
+            "sourceFiles": {
+                "Test.sol": "// SPDX-License-Identifier: MIT\r\n\r\npragma solidity 0.8.17;\r\n\r\ncontract Test {\r\n    enum E {\r\n        V1, V2, V3, V4\r\n    }\r\n    struct A {\r\n        E a;\r\n        uint256[] b;\r\n        B[] c;\r\n    }\r\n\r\n    struct B {\r\n        uint256 d;\r\n        uint256 e;\r\n    }\r\n\r\n    function get(uint256 x) external pure returns (A memory) {\r\n        uint256[] memory b = new uint256[](3);\r\n        b[0] = 1;\r\n        b[1] = 2;\r\n        b[2] = 3;\r\n        B[] memory c = new B[](3);\r\n        c[0] = B(1, 2);\r\n        c[1] = B(3, 4);\r\n        c[2] = B(5, 6);\r\n        return A(E.V3, b, c);\r\n    }\r\n}"
+            },
+            "abi": "[{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"x\",\"type\":\"uint256\"}],\"name\":\"get\",\"outputs\":[{\"components\":[{\"type\":\"uint8\"},{\"type\":\"uint256[]\"},{\"components\":[{\"type\":\"uint256\"},{\"type\":\"uint256\"}],\"type\":\"tuple[]\"}],\"internalType\":\"struct Test.A\",\"name\":\"\",\"type\":\"tuple\"}],\"stateMutability\":\"pure\",\"type\":\"function\"}]",
+            "constructorArguments": "0x0000000000000000000000003e5c63644e683549055b9be8653de26e0b4cd36e",
+            "matchType": "PARTIAL"
+        }
+    ],
+    "sourcifySources": [{
+        "fileName": "Test.sol",
+        "contractName": "Test",
+        "compilerVersion": "v0.8.17+commit.8df45f5f",
+        "compilerSettings": "{\"libraries\":{\"Test.sol\":{}},\"metadata\":{\"bytecodeHash\":\"ipfs\"},\"optimizer\":{\"enabled\":true,\"runs\":199},\"outputSelection\":{\"*\":{\"\":[\"ast\"],\"*\":[\"abi\",\"evm.bytecode\",\"evm.deployedBytecode\",\"evm.methodIdentifiers\"]}}}",
+        "sourceType": "SOLIDITY",
+        "sourceFiles": {
+            "Test.sol": "// SPDX-License-Identifier: MIT\r\n\r\npragma solidity 0.8.17;\r\n\r\ncontract Test {\r\n    enum E {\r\n        V1, V2, V3, V4\r\n    }\r\n    struct A {\r\n        E a;\r\n        uint256[] b;\r\n        B[] c;\r\n    }\r\n\r\n    struct B {\r\n        uint256 d;\r\n        uint256 e;\r\n    }\r\n\r\n    function get(uint256 x) external pure returns (A memory) {\r\n        uint256[] memory b = new uint256[](3);\r\n        b[0] = 1;\r\n        b[1] = 2;\r\n        b[2] = 3;\r\n        B[] memory c = new B[](3);\r\n        c[0] = B(1, 2);\r\n        c[1] = B(3, 4);\r\n        c[2] = B(5, 6);\r\n        return A(E.V3, b, c);\r\n    }\r\n}"
+        },
+        "abi": "[{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"x\",\"type\":\"uint256\"}],\"name\":\"get\",\"outputs\":[{\"components\":[{\"type\":\"uint8\"},{\"type\":\"uint256[]\"},{\"components\":[{\"type\":\"uint256\"},{\"type\":\"uint256\"}],\"type\":\"tuple[]\"}],\"internalType\":\"struct Test.A\",\"name\":\"\",\"type\":\"tuple\"}],\"stateMutability\":\"pure\",\"type\":\"function\"}]",
+        "constructorArguments": "0x0000000000000000000000003e5c63644e683549055b9be8653de26e0b4cd36e",
+        "matchType": "PARTIAL"
+    }]
+  }

apps/explorer/lib/explorer/application.ex

@@ -54,6 +54,9 @@ defmodule Explorer.Application do
       Supervisor.child_spec({Task.Supervisor, name: Explorer.MarketTaskSupervisor}, id: Explorer.MarketTaskSupervisor),
       Supervisor.child_spec({Task.Supervisor, name: Explorer.GenesisDataTaskSupervisor}, id: GenesisDataTaskSupervisor),
       Supervisor.child_spec({Task.Supervisor, name: Explorer.TaskSupervisor}, id: Explorer.TaskSupervisor),
+      Supervisor.child_spec({Task.Supervisor, name: Explorer.LookUpSmartContractSourcesTaskSupervisor},
+        id: LookUpSmartContractSourcesTaskSupervisor
+      ),
       Explorer.SmartContract.SolcDownloader,
       Explorer.SmartContract.VyperDownloader,
       {Registry, keys: :duplicate, name: Registry.ChainEvents, id: Registry.ChainEvents},
@@ -138,9 +141,7 @@ defmodule Explorer.Application do
   end
 
   defp sc_microservice_configure(process) do
-    config = Application.get_env(:explorer, Explorer.SmartContract.RustVerifierInterfaceBehaviour, [])
+    if Application.get_env(:explorer, Explorer.SmartContract.RustVerifierInterfaceBehaviour)[:eth_bytecode_db?] do
-
-    if config[:enabled] && config[:type] == "eth_bytecode_db" do
       process
     else
       []

apps/explorer/lib/explorer/chain/fetcher/look_up_smart_contract_sources_on_demand.ex

@@ -5,7 +5,6 @@ defmodule Explorer.Chain.Fetcher.LookUpSmartContractSourcesOnDemand do
 
   use GenServer
 
-  alias Explorer.Chain
   alias Explorer.Chain.{Address, Data, SmartContract}
   alias Explorer.Chain.Events.Publisher
   alias Explorer.SmartContract.EthBytecodeDBInterface
@@ -16,10 +15,16 @@ defmodule Explorer.Chain.Fetcher.LookUpSmartContractSourcesOnDemand do
 
   @cache_name :smart_contracts_sources_fetching
 
+  @cooldown_timeout 500
+
   def trigger_fetch(nil, _) do
     :ignore
   end
 
+  def trigger_fetch(address, %SmartContract{partially_verified: true}) do
+    GenServer.cast(__MODULE__, {:fetch, address})
+  end
+
   def trigger_fetch(_address, %SmartContract{}) do
     :ignore
   end
@@ -28,21 +33,20 @@ defmodule Explorer.Chain.Fetcher.LookUpSmartContractSourcesOnDemand do
     GenServer.cast(__MODULE__, {:fetch, address})
   end
 
-  defp fetch_sources(address) do
+  defp fetch_sources(address, only_full?) do
     creation_tx_input = contract_creation_input(address.hash)
 
-    with {:ok, %{"sourceType" => type} = source} <-
+    with {:ok, %{"sourceType" => type, "matchType" => match_type} = source} <-
            %{}
            |> prepare_bytecode_for_microservice(creation_tx_input, Data.to_string(address.contract_code))
-           |> EthBytecodeDBInterface.search_contract(),
+           |> EthBytecodeDBInterface.search_contract(address.hash),
+         :ok <- check_match_type(match_type, only_full?),
          {:ok, _} <- process_contract_source(type, source, address.hash) do
       Publisher.broadcast(%{smart_contract_was_verified: [address.hash]}, :on_demand)
     else
       _ ->
         false
     end
-
-    :ets.insert(@cache_name, {to_string(address.hash), DateTime.utc_now()})
   end
 
   def start_link(_) do
@@ -50,35 +54,61 @@ defmodule Explorer.Chain.Fetcher.LookUpSmartContractSourcesOnDemand do
   end
 
   @impl true
-  def init(opts) do
+  def init(_) do
     :ets.new(@cache_name, [
       :set,
       :named_table,
       :public
     ])
 
-    {:ok, opts}
+    {:ok,
+     %{
+       current_concurrency: 0,
+       max_concurrency:
+         Application.get_env(:explorer, Explorer.Chain.Fetcher.LookUpSmartContractSourcesOnDemand)[:max_concurrency]
+     }}
   end
 
   @impl true
-  def handle_cast({:fetch, address}, state) do
+  def handle_cast({:fetch, address}, %{current_concurrency: counter, max_concurrency: max_concurrency} = state)
-    if need_to_fetch_sources?(address) && check_interval(to_string(address.hash)) do
+      when counter < max_concurrency do
-      fetch_sources(address)
+    handle_fetch_request(address, state)
   end
 
+  @impl true
+  def handle_cast({:fetch, _address} = request, %{current_concurrency: _counter} = state) do
+    Process.send_after(self(), request, @cooldown_timeout)
+
     {:noreply, state}
   end
 
-  defp need_to_fetch_sources?(%Address{smart_contract: nil}), do: true
+  @impl true
+  def handle_info({:fetch, address}, %{current_concurrency: counter, max_concurrency: max_concurrency} = state)
+      when counter < max_concurrency do
+    handle_fetch_request(address, state)
+  end
 
-  defp need_to_fetch_sources?(%Address{hash: hash}) do
+  @impl true
-    case Chain.address_hash_to_one_smart_contract(hash) do
+  def handle_info({:fetch, _address} = request, state) do
-      nil ->
+    Process.send_after(self(), request, @cooldown_timeout)
-        true
+    {:noreply, state}
+  end
 
-      _ ->
+  @impl true
-        false
+  def handle_info({ref, _answer}, %{current_concurrency: counter} = state) do
+    Process.demonitor(ref, [:flush])
+    {:noreply, %{state | current_concurrency: counter - 1}}
   end
+
+  @impl true
+  def handle_info({:DOWN, _ref, :process, _pid, _reason}, %{current_concurrency: counter} = state) do
+    {:noreply, %{state | current_concurrency: counter - 1}}
+  end
+
+  defp partially_verified?(%Address{smart_contract: nil}), do: nil
+
+  defp partially_verified?(%Address{hash: hash}) do
+    SmartContract.select_partially_verified_by_address_hash(hash)
   end
 
   defp check_interval(address_string) do
@@ -101,7 +131,7 @@ defmodule Explorer.Chain.Fetcher.LookUpSmartContractSourcesOnDemand do
   end
 
   def process_contract_source("VYPER", source, address_hash) do
-    VyperPublisher.process_rust_verifier_response(source, address_hash, true, true)
+    VyperPublisher.process_rust_verifier_response(source, address_hash, true, true, true)
   end
 
   def process_contract_source("YUL", source, address_hash) do
@@ -109,4 +139,29 @@ defmodule Explorer.Chain.Fetcher.LookUpSmartContractSourcesOnDemand do
   end
 
   def process_contract_source(_, _source, _address_hash), do: false
+
+  defp check_match_type("PARTIAL", true), do: :full_match_required
+  defp check_match_type(_, _), do: :ok
+
+  defp handle_fetch_request(address, %{current_concurrency: counter} = state) do
+    need_to_check_and_partially_verified? =
+      check_interval(to_lowercase_string(address.hash)) && partially_verified?(address)
+
+    diff =
+      if is_nil(need_to_check_and_partially_verified?) || need_to_check_and_partially_verified? do
+        Task.Supervisor.async_nolink(Explorer.GenesisDataTaskSupervisor, fn ->
+          fetch_sources(address, need_to_check_and_partially_verified?)
+        end)
+
+        :ets.insert(@cache_name, {to_lowercase_string(address.hash), DateTime.utc_now()})
+
+        1
+      else
+        0
+      end
+
+    {:noreply, %{state | current_concurrency: counter + diff}}
+  end
+
+  defp to_lowercase_string(hash), do: hash |> to_string() |> String.downcase()
 end

apps/explorer/lib/explorer/chain/smart_contract.ex

@@ -953,4 +953,16 @@ defmodule Explorer.Chain.SmartContract do
   end
 
   defp abi_decode_address_output(_), do: nil
+
+  @spec select_partially_verified_by_address_hash(binary() | Hash.t(), keyword) :: boolean() | nil
+  def select_partially_verified_by_address_hash(address_hash, options \\ []) do
+    query =
+      from(
+        smart_contract in __MODULE__,
+        where: smart_contract.address_hash == ^address_hash,
+        select: smart_contract.partially_verified
+      )
+
+    Chain.select_repo(options).one(query)
+  end
 end

apps/explorer/lib/explorer/smart_contract/eth_bytecode_db_interface.ex

@@ -3,9 +3,31 @@ defmodule Explorer.SmartContract.EthBytecodeDBInterface do
     Adapter for interaction with https://github.com/blockscout/blockscout-rs/tree/main/eth-bytecode-db
   """
 
-  def search_contract(%{"bytecode" => _, "bytecodeType" => _} = body) do
+  def search_contract(%{"bytecode" => _, "bytecodeType" => _} = body, address_hash) do
+    if chain_id = Application.get_env(:block_scout_web, :chain_id) do
+      http_post_request(
+        bytecode_search_all_sources_url(),
+        Map.merge(body, %{
+          "chain" => to_string(chain_id),
+          "address" => to_string(address_hash)
+        })
+      )
+    else
       http_post_request(bytecode_search_sources_url(), body)
     end
+  end
+
+  def process_verifier_response(%{"sourcifySources" => [src | _]}) do
+    {:ok, Map.put(src, "sourcify?", true)}
+  end
+
+  def process_verifier_response(%{"ethBytecodeDbSources" => [src | _]}) do
+    {:ok, src}
+  end
+
+  def process_verifier_response(%{"ethBytecodeDbSources" => [], "sourcifySources" => []}) do
+    {:error, :no_matched_sources}
+  end
 
   def process_verifier_response(%{"sources" => [src | _]}) do
     {:ok, src}
@@ -15,7 +37,23 @@ defmodule Explorer.SmartContract.EthBytecodeDBInterface do
     {:ok, nil}
   end
 
-  def bytecode_search_sources_url, do: "#{base_api_url()}" <> "/bytecodes/sources:search"
+  def bytecode_search_sources_url do
+    # workaround because of https://github.com/PSPDFKit-labs/bypass/issues/122
+    if Mix.env() == :test do
+      "#{base_api_url()}" <> "/bytecodes/sources_search"
+    else
+      "#{base_api_url()}" <> "/bytecodes/sources:search"
+    end
+  end
+
+  def bytecode_search_all_sources_url do
+    # workaround because of https://github.com/PSPDFKit-labs/bypass/issues/122
+    if Mix.env() == :test do
+      "#{base_api_url()}" <> "/bytecodes/sources_search_all"
+    else
+      "#{base_api_url()}" <> "/bytecodes/sources:search-all"
+    end
+  end
 
   use Explorer.SmartContract.RustVerifierInterfaceBehaviour
 end

apps/explorer/lib/explorer/smart_contract/helper.ex

@@ -115,10 +115,16 @@ defmodule Explorer.SmartContract.Helper do
   end
 
   def cast_libraries(map) do
+    map |> Map.values() |> List.first() |> cast_libraries(map)
+  end
+
+  def cast_libraries(value, map) when is_map(value),
+    do:
       map
       |> Map.values()
       |> Enum.reduce(%{}, fn map, acc -> Map.merge(acc, map) end)
-  end
+
+  def cast_libraries(_value, map), do: map
 
   def contract_creation_input(address_hash) do
     case Chain.smart_contract_creation_tx_bytecode(address_hash) do

apps/explorer/lib/explorer/smart_contract/rust_verifier_interface_behaviour.ex

@@ -9,7 +9,7 @@ defmodule Explorer.SmartContract.RustVerifierInterfaceBehaviour do
       alias HTTPoison.Response
       require Logger
 
-      @post_timeout :timer.seconds(120)
+      @post_timeout :timer.minutes(5)
       @request_error_msg "Error while sending request to verification microservice"
 
       def verify_multi_part(

apps/explorer/lib/explorer/smart_contract/solidity/publish_helper.ex

@@ -4,9 +4,9 @@ defmodule Explorer.SmartContract.Solidity.PublishHelper do
   """
 
   alias Ecto.Changeset
-  alias Explorer.Chain
+  alias Explorer.Chain.{Address, SmartContract}
   alias Explorer.Chain.Events.Publisher, as: EventsPublisher
-  alias Explorer.Chain.SmartContract
+  alias Explorer.Chain.Fetcher.LookUpSmartContractSourcesOnDemand
   alias Explorer.SmartContract.Solidity.Publisher
   alias Explorer.ThirdPartyIntegrations.Sourcify
 
@@ -148,26 +148,25 @@ defmodule Explorer.SmartContract.Solidity.PublishHelper do
   end
 
   def check_and_verify(address_hash_string) do
-    if Chain.smart_contract_fully_verified?(address_hash_string) do
+    if Application.get_env(:explorer, Explorer.SmartContract.RustVerifierInterfaceBehaviour)[:eth_bytecode_db?] do
-      {:ok, :already_fully_verified}
+      LookUpSmartContractSourcesOnDemand.trigger_fetch(%Address{hash: address_hash_string}, nil)
     else
-      check_and_verify_inner(address_hash_string)
-    end
-  end
-
-  defp check_and_verify_inner(address_hash_string) do
       if Application.get_env(:explorer, Explorer.ThirdPartyIntegrations.Sourcify)[:enabled] do
-      if Chain.smart_contract_verified?(address_hash_string) do
+        {:error, :sourcify_disabled}
-        check_by_address_in_sourcify_if_contract_verified(address_hash_string)
       else
-        check_by_address_in_sourcify_else(address_hash_string)
+        check_by_address_in_sourcify(
+          SmartContract.select_partially_verified_by_address_hash(address_hash_string),
+          address_hash_string
+        )
+      end
     end
-    else
-      {:error, :sourcify_disabled}
   end
+
+  defp check_by_address_in_sourcify(false, _address_hash_string) do
+    {:ok, :already_fully_verified}
   end
 
-  defp check_by_address_in_sourcify_if_contract_verified(address_hash_string) do
+  defp check_by_address_in_sourcify(true, address_hash_string) do
     case Sourcify.check_by_address(address_hash_string) do
       {:ok, _verified_status} ->
         get_metadata_and_publish(address_hash_string, nil)
@@ -177,7 +176,7 @@ defmodule Explorer.SmartContract.Solidity.PublishHelper do
     end
   end
 
-  defp check_by_address_in_sourcify_else(address_hash_string) do
+  defp check_by_address_in_sourcify(nil, address_hash_string) do
     case Sourcify.check_by_address_any(address_hash_string) do
       {:ok, "full", metadata} ->
         process_metadata_and_publish(address_hash_string, metadata, false, false)

apps/explorer/lib/explorer/smart_contract/solidity/publisher.ex

@@ -145,7 +145,7 @@ defmodule Explorer.SmartContract.Solidity.Publisher do
           "sourceFiles" => sources,
           "compilerSettings" => compiler_settings_string,
           "matchType" => match_type
-        },
+        } = source,
         address_hash,
         is_standard_json?,
         save_file_path?,
@@ -177,6 +177,7 @@ defmodule Explorer.SmartContract.Solidity.Publisher do
       |> Map.put("compiler_settings", if(is_standard_json?, do: compiler_settings))
       |> Map.put("partially_verified", match_type == "PARTIAL")
       |> Map.put("verified_via_eth_bytecode_db", automatically_verified?)
+      |> Map.put("verified_via_sourcify", source["sourcify?"])
 
     publish_smart_contract(address_hash, prepared_params, Jason.decode!(abi_string || "null"))
   end

apps/explorer/test/explorer/chain_test.exs

@@ -4242,14 +4242,18 @@ defmodule Explorer.ChainTest do
       assert sc_before_call.partially_verified == Map.get(valid_attrs, :partially_verified)
 
       assert {:ok, %SmartContract{}} =
-               Chain.update_smart_contract(%{address_hash: address.hash, partially_verified: false})
+               Chain.update_smart_contract(%{
+                 address_hash: address.hash,
+                 partially_verified: false,
+                 contract_source_code: "new code"
+               })
 
       sc_after_call = Repo.get_by(SmartContract, address_hash: address.hash)
       assert sc_after_call.name == Map.get(valid_attrs, :name)
       assert sc_after_call.partially_verified == false
       assert sc_after_call.compiler_version == Map.get(valid_attrs, :compiler_version)
       assert sc_after_call.optimization == Map.get(valid_attrs, :optimization)
-      assert sc_after_call.contract_source_code == Map.get(valid_attrs, :contract_source_code)
+      assert sc_after_call.contract_source_code == "new code"
     end
 
     test "check nothing changed", %{valid_attrs: valid_attrs, address: address} do

config/runtime.exs

@@ -359,11 +359,15 @@ config :explorer, Explorer.ThirdPartyIntegrations.Sourcify,
   chain_id: System.get_env("CHAIN_ID"),
   repo_url: System.get_env("SOURCIFY_REPO_URL") || "https://repo.sourcify.dev/contracts"
 
+enabled? = ConfigHelper.parse_bool_env_var("MICROSERVICE_SC_VERIFIER_ENABLED")
+# or "eth_bytecode_db"
+type = System.get_env("MICROSERVICE_SC_VERIFIER_TYPE", "sc_verifier")
+
 config :explorer, Explorer.SmartContract.RustVerifierInterfaceBehaviour,
   service_url: System.get_env("MICROSERVICE_SC_VERIFIER_URL") || "https://eth-bytecode-db.services.blockscout.com/",
-  enabled: ConfigHelper.parse_bool_env_var("MICROSERVICE_SC_VERIFIER_ENABLED"),
+  enabled: enabled?,
-  # or "eth_bytecode_db"
+  type: type,
-  type: System.get_env("MICROSERVICE_SC_VERIFIER_TYPE", "eth_bytecode_db")
+  eth_bytecode_db?: enabled? && type == "eth_bytecode_db"
 
 config :explorer, Explorer.Visualize.Sol2uml,
   service_url: System.get_env("MICROSERVICE_VISUALIZE_SOL2UML_URL"),
@@ -407,7 +411,8 @@ config :explorer, Explorer.Chain.Cache.TransactionActionTokensData,
   max_cache_size: ConfigHelper.parse_integer_env_var("INDEXER_TX_ACTIONS_MAX_TOKEN_CACHE_SIZE", 100_000)
 
 config :explorer, Explorer.Chain.Fetcher.LookUpSmartContractSourcesOnDemand,
-  fetch_interval: ConfigHelper.parse_time_env_var("MICROSERVICE_ETH_BYTECODE_DB_INTERVAL_BETWEEN_LOOKUPS", "10m")
+  fetch_interval: ConfigHelper.parse_time_env_var("MICROSERVICE_ETH_BYTECODE_DB_INTERVAL_BETWEEN_LOOKUPS", "10m"),
+  max_concurrency: ConfigHelper.parse_integer_env_var("MICROSERVICE_ETH_BYTECODE_DB_MAX_LOOKUPS_CONCURRENCY", 10)
 
 config :explorer, Explorer.Chain.Cache.MinMissingBlockNumber,
   enabled: !ConfigHelper.parse_bool_env_var("DISABLE_INDEXER")

docker-compose/envs/common-blockscout.env

@@ -203,6 +203,7 @@ MICROSERVICE_SC_VERIFIER_ENABLED=true
 MICROSERVICE_SC_VERIFIER_URL=https://eth-bytecode-db.services.blockscout.com/
 MICROSERVICE_SC_VERIFIER_TYPE=eth_bytecode_db
 MICROSERVICE_ETH_BYTECODE_DB_INTERVAL_BETWEEN_LOOKUPS=10m
+MICROSERVICE_ETH_BYTECODE_DB_MAX_LOOKUPS_CONCURRENCY=10
 MICROSERVICE_VISUALIZE_SOL2UML_ENABLED=true
 MICROSERVICE_VISUALIZE_SOL2UML_URL=http://visualizer:8050/
 MICROSERVICE_SIG_PROVIDER_ENABLED=true