Merge pull request #1966 from poanetwork/contract-filter-performance

fix: add fields for contract filter performance
pull/2008/head
Victor Baranov 6 years ago committed by GitHub
commit 8ed8640833
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 1
      CHANGELOG.md
  2. 4
      apps/block_scout_web/lib/block_scout_web/controllers/api/rpc/contract_controller.ex
  3. 19
      apps/block_scout_web/lib/block_scout_web/etherscan.ex
  4. 12
      apps/block_scout_web/lib/block_scout_web/views/api/rpc/contract_view.ex
  5. 28
      apps/block_scout_web/test/block_scout_web/controllers/api/rpc/contract_controller_test.exs
  6. 20
      apps/block_scout_web/test/block_scout_web/controllers/api/v1/decompiled_smart_contract_controller_test.exs
  7. 137
      apps/explorer/lib/explorer/chain.ex
  8. 4
      apps/explorer/lib/explorer/chain/address.ex
  9. 14
      apps/explorer/lib/explorer/chain/import/runner/addresses.ex
  10. 2
      apps/explorer/lib/explorer/smart_contract/solc_downloader.ex
  11. 14
      apps/explorer/priv/repo/migrations/20190516140202_add_address_hash_index_to_decompiled_smart_contracts.exs
  12. 18
      apps/explorer/priv/repo/migrations/20190516160535_add_decompiled_and_verified_flag_to_addresses.exs
  13. 6
      apps/explorer/test/explorer/chain_test.exs
  14. 5
      apps/explorer/test/support/factory.ex

@ -55,6 +55,7 @@
- [#1960](https://github.com/poanetwork/blockscout/pull/1960) - do not remove bold text in decompiled contacts
- [#1917](https://github.com/poanetwork/blockscout/pull/1917) - Force block refetch if transaction is re-collated in a different block
- [#1992](https://github.com/poanetwork/blockscout/pull/1992) - fix: support https for wobserver polling
- [#1966](https://github.com/poanetwork/blockscout/pull/1966) - fix: add fields for contract filter performance
### Chore

@ -98,10 +98,10 @@ defmodule BlockScoutWeb.API.RPC.ContractController do
Chain.list_decompiled_contracts(page_size, offset, not_decompiled_with_version)
:unverified ->
Chain.list_unverified_contracts(page_size, offset)
Chain.list_unordered_unverified_contracts(page_size, offset)
:not_decompiled ->
Chain.list_not_decompiled_contracts(page_size, offset)
Chain.list_unordered_not_decompiled_contracts(page_size, offset)
:empty ->
Chain.list_empty_contracts(page_size, offset)

@ -862,11 +862,6 @@ defmodule BlockScoutWeb.Etherscan do
name: "Contract",
fields: %{
"Address" => @address_hash_type,
"DecompilerVersion" => %{
type: "decompiler version",
definition: "When decompiled source code is present, the decompiler version with which it was generated.",
example: "decompiler.version"
},
"ABI" => %{
type: "ABI",
definition: "JSON string for the contract's Application Binary Interface (ABI)",
@ -938,9 +933,16 @@ defmodule BlockScoutWeb.Etherscan do
"""
}
@contract_decompiler_version_type %{
type: "decompiler version",
definition: "When decompiled source code is present, the decompiler version with which it was generated.",
example: "decompiler.version"
}
@contract_with_sourcecode_model @contract_model
|> put_in([:fields, "SourceCode"], @contract_source_code_type)
|> put_in([:fields, "DecompiledSourceCode"], @contract_decompiled_source_code_type)
|> put_in([:fields, "DecompilerVersion"], @contract_decompiler_version_type)
@transaction_receipt_status_model %{
name: "TransactionReceiptStatus",
@ -1831,7 +1833,12 @@ defmodule BlockScoutWeb.Etherscan do
@contract_listcontracts_action %{
name: "listcontracts",
description: "Get a list of contracts, sorted ascending by the time they were first seen by the explorer.",
description: """
Get a list of contracts, sorted ascending by the time they were first seen by the explorer.
If you provide the filters `not_decompiled`(`4`) or `not_verified(4)` the results will not
be sorted for performance reasons.
""",
required_params: [],
optional_params: [
%{

@ -76,16 +76,12 @@ defmodule BlockScoutWeb.API.RPC.ContractView do
defp prepare_contract(%Address{
hash: hash,
smart_contract: nil,
decompiled_smart_contracts: decompiled_smart_contracts
smart_contract: nil
}) do
decompiled_smart_contract = latest_decompiled_smart_contract(decompiled_smart_contracts)
%{
"Address" => to_string(hash),
"ABI" => "Contract source code not verified",
"ContractName" => "",
"DecompilerVersion" => decompiler_version(decompiled_smart_contract),
"CompilerVersion" => "",
"OptimizationUsed" => ""
}
@ -93,16 +89,12 @@ defmodule BlockScoutWeb.API.RPC.ContractView do
defp prepare_contract(%Address{
hash: hash,
smart_contract: %SmartContract{} = contract,
decompiled_smart_contracts: decompiled_smart_contracts
smart_contract: %SmartContract{} = contract
}) do
decompiled_smart_contract = latest_decompiled_smart_contract(decompiled_smart_contracts)
%{
"Address" => to_string(hash),
"ABI" => Jason.encode!(contract.abi),
"ContractName" => contract.name,
"DecompilerVersion" => decompiler_version(decompiled_smart_contract),
"CompilerVersion" => contract.compiler_version,
"OptimizationUsed" => if(contract.optimization, do: "1", else: "0")
}

@ -47,7 +47,6 @@ defmodule BlockScoutWeb.API.RPC.ContractControllerTest do
"Address" => to_string(contract.address_hash),
"CompilerVersion" => contract.compiler_version,
"ContractName" => contract.name,
"DecompilerVersion" => "",
"OptimizationUsed" => if(contract.optimization, do: "1", else: "0")
}
]
@ -70,7 +69,6 @@ defmodule BlockScoutWeb.API.RPC.ContractControllerTest do
"Address" => to_string(address.hash),
"CompilerVersion" => "",
"ContractName" => "",
"DecompilerVersion" => "",
"OptimizationUsed" => ""
}
]
@ -94,7 +92,6 @@ defmodule BlockScoutWeb.API.RPC.ContractControllerTest do
"Address" => to_string(address.hash),
"CompilerVersion" => "",
"ContractName" => "",
"DecompilerVersion" => "",
"OptimizationUsed" => ""
}
]
@ -122,7 +119,6 @@ defmodule BlockScoutWeb.API.RPC.ContractControllerTest do
"Address" => to_string(address.hash),
"CompilerVersion" => "",
"ContractName" => "",
"DecompilerVersion" => "",
"OptimizationUsed" => ""
}
]
@ -145,7 +141,6 @@ defmodule BlockScoutWeb.API.RPC.ContractControllerTest do
"ABI" => Jason.encode!(contract.abi),
"Address" => to_string(contract.address_hash),
"CompilerVersion" => contract.compiler_version,
"DecompilerVersion" => "",
"ContractName" => contract.name,
"OptimizationUsed" => if(contract.optimization, do: "1", else: "0")
}
@ -170,7 +165,6 @@ defmodule BlockScoutWeb.API.RPC.ContractControllerTest do
"Address" => to_string(decompiled_smart_contract.address_hash),
"CompilerVersion" => "",
"ContractName" => "",
"DecompilerVersion" => "test_decompiler",
"OptimizationUsed" => ""
}
]
@ -194,7 +188,6 @@ defmodule BlockScoutWeb.API.RPC.ContractControllerTest do
"Address" => to_string(smart_contract.address_hash),
"CompilerVersion" => "",
"ContractName" => "",
"DecompilerVersion" => "bizbuz",
"OptimizationUsed" => ""
}
]
@ -214,16 +207,15 @@ defmodule BlockScoutWeb.API.RPC.ContractControllerTest do
assert response["message"] == "OK"
assert response["status"] == "1"
assert response["result"] == [
%{
"ABI" => "Contract source code not verified",
"Address" => to_string(smart_contract.address_hash),
"CompilerVersion" => "",
"ContractName" => "",
"DecompilerVersion" => "bizbuz",
"OptimizationUsed" => ""
}
]
assert %{
"ABI" => "Contract source code not verified",
"Address" => to_string(smart_contract.address_hash),
"CompilerVersion" => "",
"ContractName" => "",
"OptimizationUsed" => ""
} in response["result"]
refute to_string(non_match.address_hash) in Enum.map(response["result"], &Map.get(&1, "Address"))
end
test "filtering for only not_decompiled (and by extension not verified contracts)", %{params: params, conn: conn} do
@ -245,7 +237,6 @@ defmodule BlockScoutWeb.API.RPC.ContractControllerTest do
"Address" => to_string(contract_address.hash),
"CompilerVersion" => "",
"ContractName" => "",
"DecompilerVersion" => "",
"OptimizationUsed" => ""
}
]
@ -274,7 +265,6 @@ defmodule BlockScoutWeb.API.RPC.ContractControllerTest do
"Address" => to_string(contract_address.hash),
"CompilerVersion" => "",
"ContractName" => "",
"DecompilerVersion" => "",
"OptimizationUsed" => ""
}
]

@ -2,7 +2,7 @@ defmodule BlockScoutWeb.API.V1.DecompiledControllerTest do
use BlockScoutWeb.ConnCase
alias Explorer.Repo
alias Explorer.Chain.DecompiledSmartContract
alias Explorer.Chain.{Address, DecompiledSmartContract}
import Ecto.Query,
only: [from: 2]
@ -87,6 +87,24 @@ defmodule BlockScoutWeb.API.V1.DecompiledControllerTest do
assert decompiled_smart_contract.decompiler_version == decompiler_version
assert decompiled_smart_contract.decompiled_source_code == decompiled_source_code
end
test "updates the address to be decompiled", %{conn: conn} do
address_hash = to_string(insert(:address, hash: "0x0000000000000000000000000000000000000001").hash)
decompiler_version = "test_decompiler"
decompiled_source_code = "hello world"
params = %{
"address_hash" => address_hash,
"decompiler_version" => decompiler_version,
"decompiled_source_code" => decompiled_source_code
}
request = post(conn, api_v1_decompiled_smart_contract_path(conn, :create), params)
assert request.status == 201
assert Repo.get!(Address, address_hash).decompiled
end
end
describe "when user is not authorized" do

@ -554,9 +554,19 @@ defmodule Explorer.Chain do
@spec create_decompiled_smart_contract(map()) :: {:ok, Address.t()} | {:error, Ecto.Changeset.t()}
def create_decompiled_smart_contract(attrs) do
%DecompiledSmartContract{}
|> DecompiledSmartContract.changeset(attrs)
|> Repo.insert(on_conflict: :replace_all, conflict_target: [:decompiler_version, :address_hash])
changeset = DecompiledSmartContract.changeset(%DecompiledSmartContract{}, attrs)
Multi.new()
|> Multi.insert(:decompiled_smart_contract, changeset,
on_conflict: :replace_all,
conflict_target: [:decompiler_version, :address_hash]
)
|> Multi.run(:set_address_decompiled, &set_address_decompiled/2)
|> Repo.transaction()
|> case do
{:ok, %{decompiled_smart_contract: decompiled_smart_contract}} -> {:ok, decompiled_smart_contract}
{:error, _, error_value, _} -> {:error, error_value}
end
end
@doc """
@ -2235,6 +2245,7 @@ defmodule Explorer.Chain do
|> Multi.insert(:smart_contract, smart_contract_changeset)
|> Multi.run(:clear_primary_address_names, &clear_primary_address_names/2)
|> Multi.run(:insert_address_name, &create_address_name/2)
|> Multi.run(:set_address_verified, &set_address_verified/2)
|> Repo.transaction()
with {:ok, %{smart_contract: smart_contract}} <- insert_result do
@ -2242,6 +2253,35 @@ defmodule Explorer.Chain do
else
{:error, :smart_contract, changeset, _} ->
{:error, changeset}
{:error, :set_address_verified, message, _} ->
{:error, message}
end
end
defp set_address_verified(repo, %{smart_contract: %SmartContract{address_hash: address_hash}}) do
query =
from(
address in Address,
where: address.hash == ^address_hash
)
case repo.update_all(query, set: [verified: true]) do
{1, _} -> {:ok, []}
_ -> {:error, "There was an error annotating that the address has been verified."}
end
end
defp set_address_decompiled(repo, %{decompiled_smart_contract: %DecompiledSmartContract{address_hash: address_hash}}) do
query =
from(
address in Address,
where: address.hash == ^address_hash
)
case repo.update_all(query, set: [decompiled: true]) do
{1, _} -> {:ok, []}
_ -> {:error, "There was an error annotating that the address has been verified."}
end
end
@ -2754,50 +2794,46 @@ defmodule Explorer.Chain do
query =
from(
address in Address,
where:
fragment(
"EXISTS (SELECT 1 FROM decompiled_smart_contracts WHERE decompiled_smart_contracts.address_hash = ?)",
address.hash
),
preload: [:decompiled_smart_contracts, :smart_contract],
order_by: [asc: address.inserted_at],
where: address.contract_code != <<>>,
where: not is_nil(address.contract_code),
where: address.decompiled == true,
limit: ^limit,
offset: ^offset
offset: ^offset,
order_by: [asc: address.inserted_at],
preload: [:smart_contract]
)
query
|> filter_decompiled_with_version(not_decompiled_with_version)
|> reject_decompiled_with_version(not_decompiled_with_version)
|> Repo.all()
end
defp filter_decompiled_with_version(query, nil) do
query
end
defp reject_decompiled_with_version(query, nil), do: query
defp filter_decompiled_with_version(query, not_decompiled_with_version) do
from(address in query,
left_join: decompiled_smart_contract in DecompiledSmartContract,
on: decompiled_smart_contract.decompiler_version == ^not_decompiled_with_version,
on: decompiled_smart_contract.address_hash == address.hash,
where: is_nil(decompiled_smart_contract.id),
distinct: [address.hash]
defp reject_decompiled_with_version(query, reject_version) do
from(
address in query,
left_join: decompiled_smart_contract in assoc(address, :decompiled_smart_contracts),
on: decompiled_smart_contract.decompiler_version == ^reject_version,
where: is_nil(decompiled_smart_contract.address_hash)
)
end
def list_verified_contracts(limit, offset) do
query =
from(
address in Address,
where: not is_nil(address.contract_code),
join: smart_contract in SmartContract,
on: smart_contract.address_hash == address.hash,
preload: [{:smart_contract, smart_contract}, :decompiled_smart_contracts],
order_by: [asc: address.inserted_at],
smart_contract in SmartContract,
order_by: [asc: smart_contract.inserted_at],
limit: ^limit,
offset: ^offset
offset: ^offset,
preload: [:address]
)
Repo.all(query)
query
|> Repo.all()
|> Enum.map(fn smart_contract ->
Map.put(smart_contract.address, :smart_contract, smart_contract)
end)
end
def list_contracts(limit, offset) do
@ -2805,7 +2841,7 @@ defmodule Explorer.Chain do
from(
address in Address,
where: not is_nil(address.contract_code),
preload: [:smart_contract, :decompiled_smart_contracts],
preload: [:smart_contract],
order_by: [asc: address.inserted_at],
limit: ^limit,
offset: ^offset
@ -2814,22 +2850,22 @@ defmodule Explorer.Chain do
Repo.all(query)
end
def list_unverified_contracts(limit, offset) do
def list_unordered_unverified_contracts(limit, offset) do
query =
from(
address in Address,
left_join: smart_contract in SmartContract,
on: smart_contract.address_hash == address.hash,
where: not is_nil(address.contract_code),
where: is_nil(smart_contract.address_hash),
where: address.contract_code != <<>>,
preload: [{:smart_contract, smart_contract}, :decompiled_smart_contracts],
order_by: [asc: address.inserted_at],
where: not is_nil(address.contract_code),
where: fragment("? IS NOT TRUE", address.verified),
limit: ^limit,
offset: ^offset
)
Repo.all(query)
query
|> Repo.all()
|> Enum.map(fn address ->
%{address | smart_contract: nil}
end)
end
def list_empty_contracts(limit, offset) do
@ -2845,30 +2881,23 @@ defmodule Explorer.Chain do
Repo.all(query)
end
def list_not_decompiled_contracts(limit, offset) do
def list_unordered_not_decompiled_contracts(limit, offset) do
query =
from(
address in Address,
where:
fragment(
"NOT EXISTS (SELECT 1 FROM decompiled_smart_contracts WHERE decompiled_smart_contracts.address_hash = ?)",
address.hash
),
where: fragment("? IS NOT TRUE", address.verified),
where: fragment("? IS NOT TRUE", address.decompiled),
where: address.contract_code != <<>>,
left_join: smart_contract in SmartContract,
on: smart_contract.address_hash == address.hash,
left_join: decompiled_smart_contract in DecompiledSmartContract,
on: decompiled_smart_contract.address_hash == address.hash,
preload: [:smart_contract, :decompiled_smart_contracts],
where: not is_nil(address.contract_code),
where: is_nil(smart_contract.address_hash),
where: is_nil(decompiled_smart_contract.address_hash),
order_by: [asc: address.inserted_at],
limit: ^limit,
offset: ^offset
)
Repo.all(query)
query
|> Repo.all()
|> Enum.map(fn address ->
%{address | smart_contract: nil}
end)
end
@doc """

@ -22,7 +22,7 @@ defmodule Explorer.Chain.Address do
Wei
}
@optional_attrs ~w(contract_code fetched_coin_balance fetched_coin_balance_block_number nonce)a
@optional_attrs ~w(contract_code fetched_coin_balance fetched_coin_balance_block_number nonce decompiled verified)a
@required_attrs ~w(hash)a
@allowed_attrs @optional_attrs ++ @required_attrs
@ -75,6 +75,8 @@ defmodule Explorer.Chain.Address do
field(:fetched_coin_balance_block_number, :integer)
field(:contract_code, Data)
field(:nonce, :integer)
field(:decompiled, :boolean, default: false)
field(:verified, :boolean, default: false)
field(:has_decompiled_code?, :boolean, virtual: true)
field(:stale?, :boolean, virtual: true)

@ -13,6 +13,11 @@ defmodule Explorer.Chain.Import.Runner.Addresses do
@behaviour Import.Runner
@row_defaults %{
decompiled: false,
verified: false
}
# milliseconds
@timeout 60_000
@ -45,9 +50,16 @@ defmodule Explorer.Chain.Import.Runner.Addresses do
update_transactions_options = %{timeout: transactions_timeout, timestamps: timestamps}
changes_list_with_defaults =
Enum.map(changes_list, fn change ->
Enum.reduce(@row_defaults, change, fn {default_key, default_value}, acc ->
Map.put_new(acc, default_key, default_value)
end)
end)
multi
|> Multi.run(:addresses, fn repo, _ ->
insert(repo, changes_list, insert_options)
insert(repo, changes_list_with_defaults, insert_options)
end)
|> Multi.run(:created_address_code_indexed_at_transactions, fn repo, %{addresses: addresses}
when is_list(addresses) ->

@ -86,7 +86,7 @@ defmodule Explorer.SmartContract.SolcDownloader do
download_path = "https://ethereum.github.io/solc-bin/bin/soljson-#{version}.js"
download_path
|> HTTPoison.get!([], timeout: 60_000)
|> HTTPoison.get!([], timeout: 60_000, recv_timeout: 60_000)
|> Map.get(:body)
end
end

@ -0,0 +1,14 @@
defmodule Explorer.Repo.Migrations.AddAddressHashIndexToDecompiledSmartContracts do
use Ecto.Migration
def change do
execute(
"""
CREATE INDEX IF NOT EXISTS decompiled_smart_contracts_address_hash_index ON decompiled_smart_contracts(address_hash);
""",
"""
DROP INDEX IF EXISTS decompiled_smart_contracts_address_hash_index
"""
)
end
end

@ -0,0 +1,18 @@
defmodule Explorer.Repo.Migrations.AddDecompiledAndVerifiedFlagToAddresses do
use Ecto.Migration
def change do
execute(
"""
ALTER TABLE addresses
ADD COLUMN IF NOT EXISTS decompiled BOOLEAN,
ADD COLUMN IF NOT EXISTS verified BOOLEAN;
""",
"""
ALTER TABLE addresses
DROP COLUMN IF EXISTS decompiled,
DROP COLUMN IF EXISTS verified;
"""
)
end
end

@ -2859,6 +2859,12 @@ defmodule Explorer.ChainTest do
assert {:ok, _} = Chain.create_smart_contract(attrs)
assert Repo.get_by(Address.Name, name: "SimpleStorage")
end
test "sets the address verified field to true", %{valid_attrs: valid_attrs} do
assert {:ok, %SmartContract{} = smart_contract} = Chain.create_smart_contract(valid_attrs)
assert Repo.get_by(Address, hash: smart_contract.address_hash).verified == true
end
end
describe "stream_unfetched_balances/2" do

@ -482,6 +482,7 @@ defmodule Explorer.Factory do
address = %Address{
hash: address_hash(),
verified: true,
contract_code: contract_code_info().bytecode,
smart_contract: smart_contract
}
@ -519,7 +520,7 @@ defmodule Explorer.Factory do
contract_code_info = contract_code_info()
%SmartContract{
address_hash: insert(:address, contract_code: contract_code_info.bytecode).hash,
address_hash: insert(:address, contract_code: contract_code_info.bytecode, verified: true).hash,
compiler_version: contract_code_info.version,
name: contract_code_info.name,
contract_source_code: contract_code_info.source_code,
@ -532,7 +533,7 @@ defmodule Explorer.Factory do
contract_code_info = contract_code_info()
%DecompiledSmartContract{
address_hash: insert(:address, contract_code: contract_code_info.bytecode).hash,
address_hash: insert(:address, contract_code: contract_code_info.bytecode, decompiled: true).hash,
decompiler_version: "test_decompiler",
decompiled_source_code: contract_code_info.source_code
}

Loading…
Cancel
Save