CipherLabs
/
blockscout
mirror of https://github.com/blockscout/blockscout
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

perf: optimize advanced filters (#10463)

Browse Source 
* perf: optimize advanced filters

* Fix filters order

* Fix order by

* Fix block_number filtering

* Fix filters order and union

* Fix tests

* Fix: remove excessive limit

* Add internal transaction to_address_hash index

* Optimize amount filter

* Fix filtering after limit

* Some fixes

Fix address filtering;
Fix token transfers;
Fix methods search

* Remove migration; Fix query inclusion

* Optimize internal transactions query

* Fix @vbaranov review

* Fix renaming issues

* Rename function arguments

---------

Co-authored-by: Viktor Baranov <baranov.viktor.27@gmail.com>
pull/11008/head
Maxim Filonov 1 month ago committed by GitHub
parent aa3defae18
commit 930c481959
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 1102 additions and 300 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 12
      apps/block_scout_web/lib/block_scout_web/controllers/api/v2/advanced_filter_controller.ex
  2. 7
      apps/block_scout_web/lib/block_scout_web/controllers/api/v2/fallback_controller.ex
  3. 249
      apps/block_scout_web/test/block_scout_web/controllers/api/v2/advanced_filter_controller_test.exs
  4. 9
      apps/explorer/lib/explorer/chain.ex
  5. 1125
      apps/explorer/lib/explorer/chain/advanced_filter.ex

12
apps/block_scout_web/lib/block_scout_web/controllers/api/v2/advanced_filter_controller.ex
Unescape View File

@ -90,6 +90,7 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterController do
|> Keyword.update(:paging_options, %PagingOptions{page_size: CSVHelper.limit()}, fn paging_options ->
%PagingOptions{paging_options | page_size: CSVHelper.limit()}
end)
|> Keyword.put(:timeout, :timer.minutes(5))
full_options
|> AdvancedFilter.list()
@ -130,8 +131,12 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterController do
ContractMethod.find_contract_method_by_name(query, @api_true)
end
with {:method, %ContractMethod{abi: %{"name" => name}, identifier: identifier}} <- {:method, mb_contract_method} do
render(conn, :methods, methods: [%{method_id: "0x" <> Base.encode16(identifier, case: :lower), name: name}])
case mb_contract_method do
%ContractMethod{abi: %{"name" => name}, identifier: identifier} ->
render(conn, :methods, methods: [%{method_id: "0x" <> Base.encode16(identifier, case: :lower), name: name}])
_ ->
render(conn, :methods, methods: [])
end
end
end
@ -189,7 +194,8 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterController do
defp extract_filters(params) do
[
transaction_types: prepare_transaction_types(params["transaction_types"]),
# TODO: remove when frontend is adopted to new naming
transaction_types: prepare_transaction_types(params["transaction_types"] || params["tx_types"]),
methods: params["methods"] |> prepare_methods(),
age: prepare_age(params["age_from"], params["age_to"]),
from_address_hashes:

7
apps/block_scout_web/lib/block_scout_web/controllers/api/v2/fallback_controller.ex
Unescape View File

@ -282,13 +282,6 @@ defmodule BlockScoutWeb.API.V2.FallbackController do
|> render(:message, %{message: @unverified_smart_contract})
end
def call(conn, {:method, _}) do
conn
|> put_status(:not_found)
|> put_view(ApiView)
|> render(:message, %{message: @not_found})
end
def call(conn, {:is_empty_response, true}) do
conn
|> put_status(500)

249
apps/block_scout_web/test/block_scout_web/controllers/api/v2/advanced_filter_controller_test.exs
Unescape View File

@ -3,7 +3,8 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do
import Mox
alias Explorer.Chain.{AdvancedFilter, Data}
alias Explorer.Chain.SmartContract
alias Explorer.Chain.{AdvancedFilter, Data, Hash}
alias Explorer.{Factory, TestHelper}
describe "/advanced_filters" do
@ -18,7 +19,7 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do
first_transaction = :transaction |> insert() |> with_block()
insert_list(3, :token_transfer, transaction: first_transaction)
for i <- 0..2 do
for i <- 1..3 do
insert(:internal_transaction,
transaction: first_transaction,
block_hash: first_transaction.block_hash,
@ -41,7 +42,7 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do
first_transaction = :transaction |> insert() |> with_block()
insert_list(3, :token_transfer, transaction: first_transaction)
for i <- 0..2 do
for i <- 1..3 do
insert(:internal_transaction,
transaction: first_transaction,
block_hash: first_transaction.block_hash,
@ -65,7 +66,7 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do
first_transaction = :transaction |> insert() |> with_block()
insert_list(3, :token_transfer, transaction: first_transaction)
for i <- 0..2 do
for i <- 1..3 do
insert(:internal_transaction,
transaction: first_transaction,
block_hash: first_transaction.block_hash,
@ -96,7 +97,7 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do
first_transaction = :transaction |> insert() |> with_block()
insert_list(3, :token_transfer, transaction: first_transaction)
for i <- 0..2 do
for i <- 1..3 do
insert(:internal_transaction,
transaction: first_transaction,
block_hash: first_transaction.block_hash,
@ -107,7 +108,7 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do
second_transaction = :transaction |> insert() |> with_block()
for i <- 0..49 do
for i <- 1..50 do
insert(:internal_transaction,
transaction: second_transaction,
block_hash: second_transaction.block_hash,
@ -130,13 +131,20 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do
transaction = insert(:transaction) |> with_block()
for token_type <- ~w(ERC-20 ERC-404 ERC-721 ERC-1155),
token = insert(:token, type: token_type),
_ <- 0..4 do
insert(:token_transfer, transaction: transaction, token_type: token_type)
insert(:token_transfer,
transaction: transaction,
token_type: token_type,
token: token,
token_contract_address_hash: token.contract_address_hash,
token_contract_address: token.contract_address
)
end
transaction = :transaction |> insert() |> with_block()
for i <- 0..29 do
for i <- 1..30 do
insert(:internal_transaction,
transaction: transaction,
block_hash: transaction.block_hash,
@ -201,7 +209,7 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do
{:ok, method3} = Data.cast(method_id3_string <> "ab0ba0")
{:ok, method4} = Data.cast(method_id4_string <> "ab0ba0")
for i <- 0..4 do
for i <- 1..5 do
insert(:internal_transaction,
transaction: transaction,
to_address_hash: contract_address.hash,
@ -213,7 +221,7 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do
)
end
for i <- 5..9 do
for i <- 6..10 do
insert(:internal_transaction,
transaction: transaction,
to_address_hash: contract_address.hash,
@ -257,25 +265,33 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do
end
test "filter by age", %{conn: conn} do
first_timestamp = ~U[2023-12-12 00:00:00.000000Z]
[_, tx_a, _, tx_b, _] =
for i <- 0..4 do
tx = :transaction |> insert() |> with_block(status: :ok)
for i <- 0..4 do
transaction = :transaction |> insert() |> with_block(block_timestamp: Timex.shift(first_timestamp, days: i))
insert(:internal_transaction,
transaction: tx,
index: i + 1,
block_index: i + 1,
block_hash: tx.block_hash,
block: tx.block
)
insert(:internal_transaction,
transaction: transaction,
block_hash: transaction.block_hash,
index: i,
block_index: i
)
insert(:token_transfer,
transaction: tx,
block_number: tx.block_number,
log_index: i,
block_hash: tx.block_hash,
block: tx.block
)
insert(:token_transfer, transaction: transaction, block_number: transaction.block_number, log_index: i)
end
tx
end
request =
get(conn, "/api/v2/advanced-filters", %{
"age_from" => "2023-12-14T00:00:00Z",
"age_to" => "2023-12-16T00:00:00Z"
"age_from" => DateTime.to_iso8601(tx_a.block.timestamp),
"age_to" => DateTime.to_iso8601(tx_b.block.timestamp)
})
assert response = json_response(request, 200)
@ -297,8 +313,8 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do
from_address_hash: address.hash,
from_address: address,
block_hash: transaction.block_hash,
index: i,
block_index: i
index: i + 1,
block_index: i + 1
)
insert(:token_transfer,
@ -312,8 +328,8 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do
insert(:internal_transaction,
transaction: transaction,
block_hash: transaction.block_hash,
index: i,
block_index: i
index: i + 1,
block_index: i + 1
)
insert(:token_transfer, transaction: transaction, block_number: transaction.block_number, log_index: i)
@ -341,8 +357,8 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do
from_address_hash: address.hash,
from_address: address,
block_hash: transaction.block_hash,
index: i,
block_index: i
index: i + 1,
block_index: i + 1
)
insert(:token_transfer,
@ -356,8 +372,8 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do
insert(:internal_transaction,
transaction: transaction,
block_hash: transaction.block_hash,
index: i,
block_index: i
index: i + 1,
block_index: i + 1
)
insert(:token_transfer, transaction: transaction, block_number: transaction.block_number, log_index: i)
@ -391,8 +407,8 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do
from_address_hash: address_to_include.hash,
from_address: address_to_include,
block_hash: transaction.block_hash,
index: i,
block_index: i
index: i + 1,
block_index: i + 1
)
insert(:token_transfer,
@ -406,8 +422,8 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do
insert(:internal_transaction,
transaction: transaction,
block_hash: transaction.block_hash,
index: i,
block_index: i
index: i + 1,
block_index: i + 1
)
insert(:token_transfer, transaction: transaction, block_number: transaction.block_number, log_index: i)
@ -439,8 +455,8 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do
to_address_hash: address.hash,
to_address: address,
block_hash: transaction.block_hash,
index: i,
block_index: i
index: i + 1,
block_index: i + 1
)
insert(:token_transfer,
@ -454,8 +470,8 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do
insert(:internal_transaction,
transaction: transaction,
block_hash: transaction.block_hash,
index: i,
block_index: i
index: i + 1,
block_index: i + 1
)
insert(:token_transfer, transaction: transaction, block_number: transaction.block_number, log_index: i)
@ -483,8 +499,8 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do
to_address_hash: address.hash,
to_address: address,
block_hash: transaction.block_hash,
index: i,
block_index: i
index: i + 1,
block_index: i + 1
)
insert(:token_transfer,
@ -498,8 +514,8 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do
insert(:internal_transaction,
transaction: transaction,
block_hash: transaction.block_hash,
index: i,
block_index: i
index: i + 1,
block_index: i + 1
)
insert(:token_transfer, transaction: transaction, block_number: transaction.block_number, log_index: i)
@ -533,8 +549,8 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do
to_address_hash: address_to_include.hash,
to_address: address_to_include,
block_hash: transaction.block_hash,
index: i,
block_index: i
index: i + 1,
block_index: i + 1
)
insert(:token_transfer,
@ -548,8 +564,8 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do
insert(:internal_transaction,
transaction: transaction,
block_hash: transaction.block_hash,
index: i,
block_index: i
index: i + 1,
block_index: i + 1
)
insert(:token_transfer, transaction: transaction, block_number: transaction.block_number, log_index: i)
@ -583,8 +599,8 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do
from_address_hash: from_address.hash,
from_address: from_address,
block_hash: transaction.block_hash,
index: i,
block_index: i
index: i + 1,
block_index: i + 1
)
insert(:token_transfer,
@ -603,8 +619,8 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do
to_address_hash: to_address.hash,
to_address: to_address,
block_hash: transaction.block_hash,
index: i,
block_index: i
index: i + 1,
block_index: i + 1
)
insert(:token_transfer,
@ -632,8 +648,8 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do
from_address_hash: from_address.hash,
from_address: from_address,
block_hash: transaction.block_hash,
index: i,
block_index: i
index: i + 1,
block_index: i + 1
)
insert(:token_transfer,
@ -650,8 +666,8 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do
insert(:internal_transaction,
transaction: transaction,
block_hash: transaction.block_hash,
index: i,
block_index: i
index: i + 1,
block_index: i + 1
)
insert(:token_transfer, transaction: transaction, block_number: transaction.block_number, log_index: i)
@ -686,8 +702,8 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do
from_address_hash: from_address.hash,
from_address: from_address,
block_hash: transaction.block_hash,
index: i,
block_index: i
index: i + 1,
block_index: i + 1
)
insert(:token_transfer,
@ -706,8 +722,8 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do
to_address_hash: to_address.hash,
to_address: to_address,
block_hash: transaction.block_hash,
index: i,
block_index: i
index: i + 1,
block_index: i + 1
)
insert(:token_transfer,
@ -735,8 +751,8 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do
from_address_hash: from_address.hash,
from_address: from_address,
block_hash: transaction.block_hash,
index: i,
block_index: i
index: i + 1,
block_index: i + 1
)
insert(:token_transfer,
@ -753,8 +769,8 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do
insert(:internal_transaction,
transaction: transaction,
block_hash: transaction.block_hash,
index: i,
block_index: i
index: i + 1,
block_index: i + 1
)
insert(:token_transfer, transaction: transaction, block_number: transaction.block_number, log_index: i)
@ -779,8 +795,8 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do
insert(:internal_transaction,
transaction: transaction,
block_hash: transaction.block_hash,
index: 0,
block_index: 0,
index: 1,
block_index: 1,
value: i * 10 ** 18
)
@ -908,13 +924,110 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do
assert Enum.count(response["items"]) == 3
end
test "correct query with all filters and pagination", %{conn: conn} do
for address_relation <- [:or, :and] do
method_id_string = "0xa9059cbb"
{:ok, method} = Data.cast(method_id_string <> "ab0ba0")
transaction_from_address = insert(:address)
transaction_to_address = insert(:address)
token_transfer_from_address = insert(:address)
token_transfer_to_address = insert(:address)
token = insert(:token)
{:ok, burn_address_hash} = Hash.Address.cast(SmartContract.burn_address_hash_string())
insert_list(5, :transaction)
transactions =
for _ <- 0..29 do
transaction =
insert(:transaction,
from_address: transaction_from_address,
from_address_hash: transaction_from_address.hash,
to_address: transaction_to_address,
to_address_hash: transaction_to_address.hash,
value: Enum.random(0..1_000_000),
input: method
)
|> with_block()
insert(:token_transfer,
transaction: transaction,
block_number: transaction.block_number,
amount: Enum.random(0..1_000_000),
from_address: token_transfer_from_address,
from_address_hash: token_transfer_from_address.hash,
to_address: token_transfer_to_address,
to_address_hash: token_transfer_to_address.hash,
token_contract_address: token.contract_address,
token_contract_address_hash: token.contract_address_hash
)
transaction
end
insert_list(5, :transaction)
from_timestamp = List.first(transactions).block.timestamp
to_timestamp = List.last(transactions).block.timestamp
params = %{
"tx_types" => "coin_transfer,ERC-20",
"methods" => method_id_string,
"age_from" => from_timestamp |> DateTime.to_iso8601(),
"age_to" => to_timestamp |> DateTime.to_iso8601(),
"from_address_hashes_to_include" => "#{transaction_from_address.hash},#{token_transfer_from_address.hash}",
"to_address_hashes_to_include" => "#{transaction_to_address.hash},#{token_transfer_to_address.hash}",
"address_relation" => to_string(address_relation),
"amount_from" => "0",
"amount_to" => "1000000",
"token_contract_address_hashes_to_include" => "native,#{token.contract_address_hash}",
"token_contract_address_hashes_to_exclude" => "#{burn_address_hash}"
}
request =
get(conn, "/api/v2/advanced-filters", params)
assert response = json_response(request, 200)
request_2nd_page = get(conn, "/api/v2/advanced-filters", Map.merge(params, response["next_page_params"]))
assert response_2nd_page = json_response(request_2nd_page, 200)
check_paginated_response(
AdvancedFilter.list(
tx_types: ["COIN_TRANSFER", "ERC-20"],
methods: ["0xa9059cbb"],
age: [from: from_timestamp, to: to_timestamp],
from_address_hashes: [
include: [transaction_from_address.hash, token_transfer_from_address.hash],
exclude: nil
],
to_address_hashes: [
include: [transaction_to_address.hash, token_transfer_to_address.hash],
exclude: nil
],
address_relation: address_relation,
amount: [from: Decimal.new("0"), to: Decimal.new("1000000")],
token_contract_address_hashes: [
include: [
"native",
token.contract_address_hash
],
exclude: [burn_address_hash]
],
api?: true
),
response["items"],
response_2nd_page["items"]
)
end
end
end
describe "/advanced_filters/methods?q=" do
test "returns 404 if method does not exist", %{conn: conn} do
test "returns empty list if method does not exist", %{conn: conn} do
request = get(conn, "/api/v2/advanced-filters/methods", %{"q" => "foo"})
assert response = json_response(request, 404)
assert response["message"] == "Not found"
assert response = json_response(request, 200)
assert response == []
end
test "finds method by name", %{conn: conn} do

9
apps/explorer/lib/explorer/chain.ex
Unescape View File

@ -2452,8 +2452,6 @@ defmodule Explorer.Chain do
@spec timestamp_to_block_number(DateTime.t(), :before | :after, boolean()) ::
{:ok, Block.block_number()} | {:error, :not_found}
def timestamp_to_block_number(given_timestamp, closest, from_api) do
{:ok, t} = Timex.format(given_timestamp, "%Y-%m-%d %H:%M:%S", :strftime)
consensus_blocks_query =
from(
block in Block,
@ -2463,7 +2461,7 @@ defmodule Explorer.Chain do
gt_timestamp_query =
from(
block in consensus_blocks_query,
where: fragment("? >= TO_TIMESTAMP(?, 'YYYY-MM-DD HH24:MI:SS')", block.timestamp, ^t),
where: block.timestamp >= ^given_timestamp,
order_by: [asc: block.timestamp],
limit: 1,
select: block
@ -2472,7 +2470,7 @@ defmodule Explorer.Chain do
lt_timestamp_query =
from(
block in consensus_blocks_query,
where: fragment("? <= TO_TIMESTAMP(?, 'YYYY-MM-DD HH24:MI:SS')", block.timestamp, ^t),
where: block.timestamp <= ^given_timestamp,
order_by: [desc: block.timestamp],
limit: 1,
select: block
@ -2484,8 +2482,7 @@ defmodule Explorer.Chain do
from(
block in subquery(union_query),
select: block,
order_by:
fragment("abs(extract(epoch from (? - TO_TIMESTAMP(?, 'YYYY-MM-DD HH24:MI:SS'))))", block.timestamp, ^t),
order_by: fragment("abs(extract(epoch from (? - ?)))", block.timestamp, ^given_timestamp),
limit: 1
)

1125
apps/explorer/lib/explorer/chain/advanced_filter.ex
View File

File diff suppressed because it is too large Load Diff
Write Preview
Loading…
Cancel
Save