From 930c481959409be8adf24e5bc5901f6f4009a9db Mon Sep 17 00:00:00 2001 From: Maxim Filonov <53992153+sl1depengwyn@users.noreply.github.com> Date: Tue, 22 Oct 2024 20:16:23 +0300 Subject: [PATCH] perf: optimize advanced filters (#10463) * perf: optimize advanced filters * Fix filters order * Fix order by * Fix block_number filtering * Fix filters order and union * Fix tests * Fix: remove excessive limit * Add internal transaction to_address_hash index * Optimize amount filter * Fix filtering after limit * Some fixes Fix address filtering; Fix token transfers; Fix methods search * Remove migration; Fix query inclusion * Optimize internal transactions query * Fix @vbaranov review * Fix renaming issues * Rename function arguments --------- Co-authored-by: Viktor Baranov --- .../api/v2/advanced_filter_controller.ex | 12 +- .../controllers/api/v2/fallback_controller.ex | 7 - .../v2/advanced_filter_controller_test.exs | 249 +++- apps/explorer/lib/explorer/chain.ex | 9 +- .../lib/explorer/chain/advanced_filter.ex | 1125 +++++++++++++---- 5 files changed, 1102 insertions(+), 300 deletions(-) diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/advanced_filter_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/advanced_filter_controller.ex index 922788dde6..cfb074a7f8 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/advanced_filter_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/advanced_filter_controller.ex @@ -90,6 +90,7 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterController do |> Keyword.update(:paging_options, %PagingOptions{page_size: CSVHelper.limit()}, fn paging_options -> %PagingOptions{paging_options | page_size: CSVHelper.limit()} end) + |> Keyword.put(:timeout, :timer.minutes(5)) full_options |> AdvancedFilter.list() @@ -130,8 +131,12 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterController do ContractMethod.find_contract_method_by_name(query, @api_true) end - with {:method, %ContractMethod{abi: %{"name" => name}, identifier: identifier}} <- {:method, mb_contract_method} do - render(conn, :methods, methods: [%{method_id: "0x" <> Base.encode16(identifier, case: :lower), name: name}]) + case mb_contract_method do + %ContractMethod{abi: %{"name" => name}, identifier: identifier} -> + render(conn, :methods, methods: [%{method_id: "0x" <> Base.encode16(identifier, case: :lower), name: name}]) + + _ -> + render(conn, :methods, methods: []) end end end @@ -189,7 +194,8 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterController do defp extract_filters(params) do [ - transaction_types: prepare_transaction_types(params["transaction_types"]), + # TODO: remove when frontend is adopted to new naming + transaction_types: prepare_transaction_types(params["transaction_types"] || params["tx_types"]), methods: params["methods"] |> prepare_methods(), age: prepare_age(params["age_from"], params["age_to"]), from_address_hashes: diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/fallback_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/fallback_controller.ex index 0879bd5adf..127bfe7840 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/fallback_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/fallback_controller.ex @@ -282,13 +282,6 @@ defmodule BlockScoutWeb.API.V2.FallbackController do |> render(:message, %{message: @unverified_smart_contract}) end - def call(conn, {:method, _}) do - conn - |> put_status(:not_found) - |> put_view(ApiView) - |> render(:message, %{message: @not_found}) - end - def call(conn, {:is_empty_response, true}) do conn |> put_status(500) diff --git a/apps/block_scout_web/test/block_scout_web/controllers/api/v2/advanced_filter_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/api/v2/advanced_filter_controller_test.exs index 9f85979f33..cca5e007a5 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/api/v2/advanced_filter_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/api/v2/advanced_filter_controller_test.exs @@ -3,7 +3,8 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do import Mox - alias Explorer.Chain.{AdvancedFilter, Data} + alias Explorer.Chain.SmartContract + alias Explorer.Chain.{AdvancedFilter, Data, Hash} alias Explorer.{Factory, TestHelper} describe "/advanced_filters" do @@ -18,7 +19,7 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do first_transaction = :transaction |> insert() |> with_block() insert_list(3, :token_transfer, transaction: first_transaction) - for i <- 0..2 do + for i <- 1..3 do insert(:internal_transaction, transaction: first_transaction, block_hash: first_transaction.block_hash, @@ -41,7 +42,7 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do first_transaction = :transaction |> insert() |> with_block() insert_list(3, :token_transfer, transaction: first_transaction) - for i <- 0..2 do + for i <- 1..3 do insert(:internal_transaction, transaction: first_transaction, block_hash: first_transaction.block_hash, @@ -65,7 +66,7 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do first_transaction = :transaction |> insert() |> with_block() insert_list(3, :token_transfer, transaction: first_transaction) - for i <- 0..2 do + for i <- 1..3 do insert(:internal_transaction, transaction: first_transaction, block_hash: first_transaction.block_hash, @@ -96,7 +97,7 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do first_transaction = :transaction |> insert() |> with_block() insert_list(3, :token_transfer, transaction: first_transaction) - for i <- 0..2 do + for i <- 1..3 do insert(:internal_transaction, transaction: first_transaction, block_hash: first_transaction.block_hash, @@ -107,7 +108,7 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do second_transaction = :transaction |> insert() |> with_block() - for i <- 0..49 do + for i <- 1..50 do insert(:internal_transaction, transaction: second_transaction, block_hash: second_transaction.block_hash, @@ -130,13 +131,20 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do transaction = insert(:transaction) |> with_block() for token_type <- ~w(ERC-20 ERC-404 ERC-721 ERC-1155), + token = insert(:token, type: token_type), _ <- 0..4 do - insert(:token_transfer, transaction: transaction, token_type: token_type) + insert(:token_transfer, + transaction: transaction, + token_type: token_type, + token: token, + token_contract_address_hash: token.contract_address_hash, + token_contract_address: token.contract_address + ) end transaction = :transaction |> insert() |> with_block() - for i <- 0..29 do + for i <- 1..30 do insert(:internal_transaction, transaction: transaction, block_hash: transaction.block_hash, @@ -201,7 +209,7 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do {:ok, method3} = Data.cast(method_id3_string <> "ab0ba0") {:ok, method4} = Data.cast(method_id4_string <> "ab0ba0") - for i <- 0..4 do + for i <- 1..5 do insert(:internal_transaction, transaction: transaction, to_address_hash: contract_address.hash, @@ -213,7 +221,7 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do ) end - for i <- 5..9 do + for i <- 6..10 do insert(:internal_transaction, transaction: transaction, to_address_hash: contract_address.hash, @@ -257,25 +265,33 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do end test "filter by age", %{conn: conn} do - first_timestamp = ~U[2023-12-12 00:00:00.000000Z] + [_, tx_a, _, tx_b, _] = + for i <- 0..4 do + tx = :transaction |> insert() |> with_block(status: :ok) - for i <- 0..4 do - transaction = :transaction |> insert() |> with_block(block_timestamp: Timex.shift(first_timestamp, days: i)) + insert(:internal_transaction, + transaction: tx, + index: i + 1, + block_index: i + 1, + block_hash: tx.block_hash, + block: tx.block + ) - insert(:internal_transaction, - transaction: transaction, - block_hash: transaction.block_hash, - index: i, - block_index: i - ) + insert(:token_transfer, + transaction: tx, + block_number: tx.block_number, + log_index: i, + block_hash: tx.block_hash, + block: tx.block + ) - insert(:token_transfer, transaction: transaction, block_number: transaction.block_number, log_index: i) - end + tx + end request = get(conn, "/api/v2/advanced-filters", %{ - "age_from" => "2023-12-14T00:00:00Z", - "age_to" => "2023-12-16T00:00:00Z" + "age_from" => DateTime.to_iso8601(tx_a.block.timestamp), + "age_to" => DateTime.to_iso8601(tx_b.block.timestamp) }) assert response = json_response(request, 200) @@ -297,8 +313,8 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do from_address_hash: address.hash, from_address: address, block_hash: transaction.block_hash, - index: i, - block_index: i + index: i + 1, + block_index: i + 1 ) insert(:token_transfer, @@ -312,8 +328,8 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do insert(:internal_transaction, transaction: transaction, block_hash: transaction.block_hash, - index: i, - block_index: i + index: i + 1, + block_index: i + 1 ) insert(:token_transfer, transaction: transaction, block_number: transaction.block_number, log_index: i) @@ -341,8 +357,8 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do from_address_hash: address.hash, from_address: address, block_hash: transaction.block_hash, - index: i, - block_index: i + index: i + 1, + block_index: i + 1 ) insert(:token_transfer, @@ -356,8 +372,8 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do insert(:internal_transaction, transaction: transaction, block_hash: transaction.block_hash, - index: i, - block_index: i + index: i + 1, + block_index: i + 1 ) insert(:token_transfer, transaction: transaction, block_number: transaction.block_number, log_index: i) @@ -391,8 +407,8 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do from_address_hash: address_to_include.hash, from_address: address_to_include, block_hash: transaction.block_hash, - index: i, - block_index: i + index: i + 1, + block_index: i + 1 ) insert(:token_transfer, @@ -406,8 +422,8 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do insert(:internal_transaction, transaction: transaction, block_hash: transaction.block_hash, - index: i, - block_index: i + index: i + 1, + block_index: i + 1 ) insert(:token_transfer, transaction: transaction, block_number: transaction.block_number, log_index: i) @@ -439,8 +455,8 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do to_address_hash: address.hash, to_address: address, block_hash: transaction.block_hash, - index: i, - block_index: i + index: i + 1, + block_index: i + 1 ) insert(:token_transfer, @@ -454,8 +470,8 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do insert(:internal_transaction, transaction: transaction, block_hash: transaction.block_hash, - index: i, - block_index: i + index: i + 1, + block_index: i + 1 ) insert(:token_transfer, transaction: transaction, block_number: transaction.block_number, log_index: i) @@ -483,8 +499,8 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do to_address_hash: address.hash, to_address: address, block_hash: transaction.block_hash, - index: i, - block_index: i + index: i + 1, + block_index: i + 1 ) insert(:token_transfer, @@ -498,8 +514,8 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do insert(:internal_transaction, transaction: transaction, block_hash: transaction.block_hash, - index: i, - block_index: i + index: i + 1, + block_index: i + 1 ) insert(:token_transfer, transaction: transaction, block_number: transaction.block_number, log_index: i) @@ -533,8 +549,8 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do to_address_hash: address_to_include.hash, to_address: address_to_include, block_hash: transaction.block_hash, - index: i, - block_index: i + index: i + 1, + block_index: i + 1 ) insert(:token_transfer, @@ -548,8 +564,8 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do insert(:internal_transaction, transaction: transaction, block_hash: transaction.block_hash, - index: i, - block_index: i + index: i + 1, + block_index: i + 1 ) insert(:token_transfer, transaction: transaction, block_number: transaction.block_number, log_index: i) @@ -583,8 +599,8 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do from_address_hash: from_address.hash, from_address: from_address, block_hash: transaction.block_hash, - index: i, - block_index: i + index: i + 1, + block_index: i + 1 ) insert(:token_transfer, @@ -603,8 +619,8 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do to_address_hash: to_address.hash, to_address: to_address, block_hash: transaction.block_hash, - index: i, - block_index: i + index: i + 1, + block_index: i + 1 ) insert(:token_transfer, @@ -632,8 +648,8 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do from_address_hash: from_address.hash, from_address: from_address, block_hash: transaction.block_hash, - index: i, - block_index: i + index: i + 1, + block_index: i + 1 ) insert(:token_transfer, @@ -650,8 +666,8 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do insert(:internal_transaction, transaction: transaction, block_hash: transaction.block_hash, - index: i, - block_index: i + index: i + 1, + block_index: i + 1 ) insert(:token_transfer, transaction: transaction, block_number: transaction.block_number, log_index: i) @@ -686,8 +702,8 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do from_address_hash: from_address.hash, from_address: from_address, block_hash: transaction.block_hash, - index: i, - block_index: i + index: i + 1, + block_index: i + 1 ) insert(:token_transfer, @@ -706,8 +722,8 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do to_address_hash: to_address.hash, to_address: to_address, block_hash: transaction.block_hash, - index: i, - block_index: i + index: i + 1, + block_index: i + 1 ) insert(:token_transfer, @@ -735,8 +751,8 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do from_address_hash: from_address.hash, from_address: from_address, block_hash: transaction.block_hash, - index: i, - block_index: i + index: i + 1, + block_index: i + 1 ) insert(:token_transfer, @@ -753,8 +769,8 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do insert(:internal_transaction, transaction: transaction, block_hash: transaction.block_hash, - index: i, - block_index: i + index: i + 1, + block_index: i + 1 ) insert(:token_transfer, transaction: transaction, block_number: transaction.block_number, log_index: i) @@ -779,8 +795,8 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do insert(:internal_transaction, transaction: transaction, block_hash: transaction.block_hash, - index: 0, - block_index: 0, + index: 1, + block_index: 1, value: i * 10 ** 18 ) @@ -908,13 +924,110 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do assert Enum.count(response["items"]) == 3 end + + test "correct query with all filters and pagination", %{conn: conn} do + for address_relation <- [:or, :and] do + method_id_string = "0xa9059cbb" + {:ok, method} = Data.cast(method_id_string <> "ab0ba0") + transaction_from_address = insert(:address) + transaction_to_address = insert(:address) + token_transfer_from_address = insert(:address) + token_transfer_to_address = insert(:address) + token = insert(:token) + {:ok, burn_address_hash} = Hash.Address.cast(SmartContract.burn_address_hash_string()) + + insert_list(5, :transaction) + + transactions = + for _ <- 0..29 do + transaction = + insert(:transaction, + from_address: transaction_from_address, + from_address_hash: transaction_from_address.hash, + to_address: transaction_to_address, + to_address_hash: transaction_to_address.hash, + value: Enum.random(0..1_000_000), + input: method + ) + |> with_block() + + insert(:token_transfer, + transaction: transaction, + block_number: transaction.block_number, + amount: Enum.random(0..1_000_000), + from_address: token_transfer_from_address, + from_address_hash: token_transfer_from_address.hash, + to_address: token_transfer_to_address, + to_address_hash: token_transfer_to_address.hash, + token_contract_address: token.contract_address, + token_contract_address_hash: token.contract_address_hash + ) + + transaction + end + + insert_list(5, :transaction) + + from_timestamp = List.first(transactions).block.timestamp + to_timestamp = List.last(transactions).block.timestamp + + params = %{ + "tx_types" => "coin_transfer,ERC-20", + "methods" => method_id_string, + "age_from" => from_timestamp |> DateTime.to_iso8601(), + "age_to" => to_timestamp |> DateTime.to_iso8601(), + "from_address_hashes_to_include" => "#{transaction_from_address.hash},#{token_transfer_from_address.hash}", + "to_address_hashes_to_include" => "#{transaction_to_address.hash},#{token_transfer_to_address.hash}", + "address_relation" => to_string(address_relation), + "amount_from" => "0", + "amount_to" => "1000000", + "token_contract_address_hashes_to_include" => "native,#{token.contract_address_hash}", + "token_contract_address_hashes_to_exclude" => "#{burn_address_hash}" + } + + request = + get(conn, "/api/v2/advanced-filters", params) + + assert response = json_response(request, 200) + request_2nd_page = get(conn, "/api/v2/advanced-filters", Map.merge(params, response["next_page_params"])) + assert response_2nd_page = json_response(request_2nd_page, 200) + + check_paginated_response( + AdvancedFilter.list( + tx_types: ["COIN_TRANSFER", "ERC-20"], + methods: ["0xa9059cbb"], + age: [from: from_timestamp, to: to_timestamp], + from_address_hashes: [ + include: [transaction_from_address.hash, token_transfer_from_address.hash], + exclude: nil + ], + to_address_hashes: [ + include: [transaction_to_address.hash, token_transfer_to_address.hash], + exclude: nil + ], + address_relation: address_relation, + amount: [from: Decimal.new("0"), to: Decimal.new("1000000")], + token_contract_address_hashes: [ + include: [ + "native", + token.contract_address_hash + ], + exclude: [burn_address_hash] + ], + api?: true + ), + response["items"], + response_2nd_page["items"] + ) + end + end end describe "/advanced_filters/methods?q=" do - test "returns 404 if method does not exist", %{conn: conn} do + test "returns empty list if method does not exist", %{conn: conn} do request = get(conn, "/api/v2/advanced-filters/methods", %{"q" => "foo"}) - assert response = json_response(request, 404) - assert response["message"] == "Not found" + assert response = json_response(request, 200) + assert response == [] end test "finds method by name", %{conn: conn} do diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index 27db2c00e1..04446a6e44 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -2452,8 +2452,6 @@ defmodule Explorer.Chain do @spec timestamp_to_block_number(DateTime.t(), :before | :after, boolean()) :: {:ok, Block.block_number()} | {:error, :not_found} def timestamp_to_block_number(given_timestamp, closest, from_api) do - {:ok, t} = Timex.format(given_timestamp, "%Y-%m-%d %H:%M:%S", :strftime) - consensus_blocks_query = from( block in Block, @@ -2463,7 +2461,7 @@ defmodule Explorer.Chain do gt_timestamp_query = from( block in consensus_blocks_query, - where: fragment("? >= TO_TIMESTAMP(?, 'YYYY-MM-DD HH24:MI:SS')", block.timestamp, ^t), + where: block.timestamp >= ^given_timestamp, order_by: [asc: block.timestamp], limit: 1, select: block @@ -2472,7 +2470,7 @@ defmodule Explorer.Chain do lt_timestamp_query = from( block in consensus_blocks_query, - where: fragment("? <= TO_TIMESTAMP(?, 'YYYY-MM-DD HH24:MI:SS')", block.timestamp, ^t), + where: block.timestamp <= ^given_timestamp, order_by: [desc: block.timestamp], limit: 1, select: block @@ -2484,8 +2482,7 @@ defmodule Explorer.Chain do from( block in subquery(union_query), select: block, - order_by: - fragment("abs(extract(epoch from (? - TO_TIMESTAMP(?, 'YYYY-MM-DD HH24:MI:SS'))))", block.timestamp, ^t), + order_by: fragment("abs(extract(epoch from (? - ?)))", block.timestamp, ^given_timestamp), limit: 1 ) diff --git a/apps/explorer/lib/explorer/chain/advanced_filter.ex b/apps/explorer/lib/explorer/chain/advanced_filter.ex index b8b2d910ec..1684670b3a 100644 --- a/apps/explorer/lib/explorer/chain/advanced_filter.ex +++ b/apps/explorer/lib/explorer/chain/advanced_filter.ex @@ -8,7 +8,7 @@ defmodule Explorer.Chain.AdvancedFilter do import Ecto.Query alias Explorer.{Chain, Helper, PagingOptions} - alias Explorer.Chain.{Address, Data, Hash, InternalTransaction, TokenTransfer, Transaction} + alias Explorer.Chain.{Address, Data, DenormalizationHelper, Hash, InternalTransaction, TokenTransfer, Transaction} @primary_key false typed_embedded_schema null: false do @@ -74,19 +74,31 @@ defmodule Explorer.Chain.AdvancedFilter do | token_contract_address_hashes() | Chain.paging_options() | Chain.api?() + | {:timeout, timeout()} ] @spec list(options()) :: [__MODULE__.t()] def list(options \\ []) do paging_options = Keyword.get(options, :paging_options) + timeout = Keyword.get(options, :timeout, :timer.seconds(60)) + + age = Keyword.get(options, :age) + + block_numbers_age = + [ + from: age[:from] && Chain.timestamp_to_block_number(age[:from], :after, Keyword.get(options, :api?, false)), + to: age[:to] && Chain.timestamp_to_block_number(age[:to], :before, Keyword.get(options, :api?, false)) + ] + tasks = options + |> Keyword.put(:block_numbers_age, block_numbers_age) |> queries(paging_options) - |> Enum.map(fn query -> Task.async(fn -> Chain.select_repo(options).all(query) end) end) + |> Enum.map(fn query -> Task.async(fn -> Chain.select_repo(options).all(query, timeout: timeout) end) end) tasks - |> Task.yield_many(:timer.seconds(60)) + |> Task.yield_many(timeout: timeout, on_timeout: :kill_task) |> Enum.flat_map(fn {_task, res} -> case res do {:ok, result} -> @@ -102,55 +114,57 @@ defmodule Explorer.Chain.AdvancedFilter do |> Enum.map(&to_advanced_filter/1) |> Enum.sort(&sort_function/2) |> take_page_size(paging_options) + |> Chain.select_repo(options).preload( + from_address: [:names, :smart_contract, :proxy_implementations], + to_address: [:names, :smart_contract, :proxy_implementations], + created_contract_address: [:names, :smart_contract, :proxy_implementations] + ) end defp queries(options, paging_options) do - cond do - only_transactions?(options) -> - [transactions_query(paging_options, options), internal_transactions_query(paging_options, options)] - - only_token_transfers?(options) -> - [token_transfers_query(paging_options, options)] - - true -> - [ - transactions_query(paging_options, options), - internal_transactions_query(paging_options, options), - token_transfers_query(paging_options, options) - ] - end + [] + |> maybe_add_transactions_queries(options, paging_options) + |> maybe_add_token_transfers_queries(options, paging_options) end - defp only_transactions?(options) do - transaction_types = options[:transaction_types] - tokens_to_include = options[:token_contract_address_hashes][:include] + defp maybe_add_transactions_queries(queries, options, paging_options) do + transaction_types = options[:transaction_types] || [] + tokens_to_include = options[:token_contract_address_hashes][:include] || [] + tokens_to_exclude = options[:token_contract_address_hashes][:exclude] || [] - transaction_types == ["COIN_TRANSFER"] or tokens_to_include == ["native"] + if (transaction_types == [] or "COIN_TRANSFER" in transaction_types) and + (tokens_to_include == [] or "native" in tokens_to_include) and + "native" not in tokens_to_exclude do + [transactions_query(paging_options, options), internal_transactions_query(paging_options, options) | queries] + else + queries + end end - defp only_token_transfers?(options) do - transaction_types = options[:transaction_types] - tokens_to_include = options[:token_contract_address_hashes][:include] - tokens_to_exclude = options[:token_contract_address_hashes][:exclude] + defp maybe_add_token_transfers_queries(queries, options, paging_options) do + transaction_types = options[:transaction_types] || [] + tokens_to_include = options[:token_contract_address_hashes][:include] || [] - (is_list(transaction_types) and length(transaction_types) > 0 and "COIN_TRANSFER" not in transaction_types) or - (is_list(tokens_to_include) and length(tokens_to_include) > 0 and "native" not in tokens_to_include) or - (is_list(tokens_to_exclude) and "native" in tokens_to_exclude) + if (transaction_types == [] or not (transaction_types |> Enum.reject(&(&1 == "COIN_TRANSFER")) |> Enum.empty?())) and + (tokens_to_include == [] or not (tokens_to_include |> Enum.reject(&(&1 == "native")) |> Enum.empty?())) do + [token_transfers_query(paging_options, options) | queries] + else + queries + end end defp to_advanced_filter(%Transaction{} = transaction) do + %{value: decimal_transaction_value} = transaction.value + %__MODULE__{ hash: transaction.hash, type: "coin_transfer", input: transaction.input, timestamp: transaction.block_timestamp, - from_address: transaction.from_address, from_address_hash: transaction.from_address_hash, - to_address: transaction.to_address, to_address_hash: transaction.to_address_hash, - created_contract_address: transaction.created_contract_address, created_contract_address_hash: transaction.created_contract_address_hash, - value: transaction.value.value, + value: decimal_transaction_value, fee: transaction |> Transaction.fee(:wei) |> elem(1), block_number: transaction.block_number, transaction_index: transaction.index @@ -158,18 +172,17 @@ defmodule Explorer.Chain.AdvancedFilter do end defp to_advanced_filter(%InternalTransaction{} = internal_transaction) do + %{value: decimal_internal_transaction_value} = internal_transaction.value + %__MODULE__{ hash: internal_transaction.transaction.hash, type: "coin_transfer", input: internal_transaction.input, timestamp: internal_transaction.transaction.block_timestamp, - from_address: internal_transaction.from_address, from_address_hash: internal_transaction.from_address_hash, - to_address: internal_transaction.to_address, to_address_hash: internal_transaction.to_address_hash, - created_contract_address: internal_transaction.created_contract_address, created_contract_address_hash: internal_transaction.created_contract_address_hash, - value: internal_transaction.value.value, + value: decimal_internal_transaction_value, fee: internal_transaction.transaction.gas_price && internal_transaction.gas_used && Decimal.mult(internal_transaction.transaction.gas_price.value, internal_transaction.gas_used), @@ -185,11 +198,8 @@ defmodule Explorer.Chain.AdvancedFilter do type: token_transfer.token_type, input: token_transfer.transaction.input, timestamp: token_transfer.transaction.block_timestamp, - from_address: token_transfer.from_address, from_address_hash: token_transfer.from_address_hash, - to_address: token_transfer.to_address, to_address_hash: token_transfer.to_address_hash, - created_contract_address: nil, created_contract_address_hash: nil, fee: token_transfer.transaction |> Transaction.fee(:wei) |> elem(1), token_transfer: %TokenTransfer{ @@ -247,19 +257,27 @@ defmodule Explorer.Chain.AdvancedFilter do defp transactions_query(paging_options, options) do query = - from(transaction in Transaction, - as: :transaction, - preload: [ - :block, - from_address: [:names, :smart_contract, :proxy_implementations], - to_address: [:scam_badge, :names, :smart_contract, :proxy_implementations], - created_contract_address: [:scam_badge, :names, :smart_contract, :proxy_implementations] - ], - order_by: [ - desc: transaction.block_number, - desc: transaction.index - ] - ) + if DenormalizationHelper.transactions_denormalization_finished?() do + from(transaction in Transaction, + as: :transaction, + where: transaction.block_consensus == true, + order_by: [ + desc: transaction.block_number, + desc: transaction.index + ] + ) + else + from(transaction in Transaction, + as: :transaction, + join: block in assoc(transaction, :block), + as: :block, + where: block.consensus == true, + order_by: [ + desc: transaction.block_number, + desc: transaction.index + ] + ) + end query |> page_transactions(paging_options) @@ -286,27 +304,57 @@ defmodule Explorer.Chain.AdvancedFilter do defp internal_transactions_query(paging_options, options) do query = - from(internal_transaction in InternalTransaction, - as: :internal_transaction, - join: transaction in assoc(internal_transaction, :transaction), - as: :transaction, - preload: [ - from_address: [:names, :smart_contract, :proxy_implementations], - to_address: [:scam_badge, :names, :smart_contract, :proxy_implementations], - created_contract_address: [:scam_badge, :names, :smart_contract, :proxy_implementations], - transaction: transaction - ], - order_by: [ - desc: transaction.block_number, - desc: transaction.index, - desc: internal_transaction.index - ] - ) + if DenormalizationHelper.transactions_denormalization_finished?() do + from(internal_transaction in InternalTransaction, + as: :internal_transaction, + join: transaction in assoc(internal_transaction, :transaction), + as: :transaction, + where: transaction.block_consensus == true, + where: + (internal_transaction.type == :call and internal_transaction.index > 0) or + internal_transaction.type != :call, + order_by: [ + desc: internal_transaction.block_number, + desc: internal_transaction.transaction_index, + desc: internal_transaction.index + ] + ) + else + from(internal_transaction in InternalTransaction, + as: :internal_transaction, + join: transaction in assoc(internal_transaction, :transaction), + as: :transaction, + join: block in assoc(internal_transaction, :block), + as: :block, + where: block.consensus == true, + where: + (internal_transaction.type == :call and internal_transaction.index > 0) or + internal_transaction.type != :call, + order_by: [ + desc: internal_transaction.block_number, + desc: internal_transaction.transaction_index, + desc: internal_transaction.index + ] + ) + end query |> page_internal_transactions(paging_options) |> limit_query(paging_options) - |> apply_transactions_filters(options) + |> apply_internal_transactions_filters(options) + |> limit_query(paging_options) + |> preload([:transaction]) + end + + defp page_internal_transactions(query, %PagingOptions{ + key: %{ + block_number: block_number, + transaction_index: _transaction_index, + internal_transaction_index: nil + } + }) + when block_number < 0 do + query |> where(false) end defp page_internal_transactions(query, %PagingOptions{ @@ -315,25 +363,44 @@ defmodule Explorer.Chain.AdvancedFilter do transaction_index: transaction_index, internal_transaction_index: nil } - }) do - case {block_number, transaction_index} do - {0, 0} -> - query |> where(as(:transaction).block_number == ^block_number and as(:transaction).index == ^transaction_index) + }) + when block_number > 0 and transaction_index <= 0 do + query |> where(as(:transaction).block_number < ^block_number) + end - {0, transaction_index} -> - query - |> where(as(:transaction).block_number == ^block_number and as(:transaction).index <= ^transaction_index) + defp page_internal_transactions(query, %PagingOptions{ + key: %{ + block_number: 0, + transaction_index: 0, + internal_transaction_index: nil + } + }) do + query |> where(as(:transaction).block_number == 0 and as(:transaction).index == 0) + end - {block_number, 0} -> - query |> where(as(:transaction).block_number < ^block_number) + defp page_internal_transactions(query, %PagingOptions{ + key: %{ + block_number: 0, + transaction_index: transaction_index, + internal_transaction_index: nil + } + }) do + query + |> where(as(:transaction).block_number == 0 and as(:transaction).index <= ^transaction_index) + end - _ -> - query - |> where( - as(:transaction).block_number < ^block_number or - (as(:transaction).block_number == ^block_number and as(:transaction).index <= ^transaction_index) - ) - end + defp page_internal_transactions(query, %PagingOptions{ + key: %{ + block_number: block_number, + transaction_index: transaction_index, + internal_transaction_index: nil + } + }) do + query + |> where( + as(:transaction).block_number < ^block_number or + (as(:transaction).block_number == ^block_number and as(:transaction).index <= ^transaction_index) + ) end defp page_internal_transactions(query, %PagingOptions{ @@ -358,68 +425,148 @@ defmodule Explorer.Chain.AdvancedFilter do defp token_transfers_query(paging_options, options) do token_transfer_query = - from(token_transfer in TokenTransfer, - as: :token_transfer, - join: transaction in assoc(token_transfer, :transaction), - as: :transaction, - join: token in assoc(token_transfer, :token), - as: :token, - select: %TokenTransfer{ - token_transfer - | token_id: fragment("UNNEST(?)", token_transfer.token_ids), - amount: - fragment("UNNEST(COALESCE(?, ARRAY[COALESCE(?, 1)]))", token_transfer.amounts, token_transfer.amount), - reverse_index_in_batch: - fragment("GENERATE_SERIES(COALESCE(ARRAY_LENGTH(?, 1), 1), 1, -1)", token_transfer.amounts), - token_decimals: token.decimals - }, - order_by: [ - desc: token_transfer.block_number, - desc: token_transfer.log_index - ] - ) + if DenormalizationHelper.transactions_denormalization_finished?() do + from(token_transfer in TokenTransfer, + as: :token_transfer, + join: transaction in assoc(token_transfer, :transaction), + as: :transaction, + join: token in assoc(token_transfer, :token), + as: :token, + select: %TokenTransfer{ + token_transfer + | token_id: fragment("UNNEST(?)", token_transfer.token_ids), + amount: + fragment("UNNEST(COALESCE(?, ARRAY[COALESCE(?, 1)]))", token_transfer.amounts, token_transfer.amount), + reverse_index_in_batch: + fragment("GENERATE_SERIES(COALESCE(ARRAY_LENGTH(?, 1), 1), 1, -1)", token_transfer.amounts), + token_decimals: token.decimals + }, + where: transaction.block_consensus == true, + order_by: [ + desc: token_transfer.block_number, + desc: token_transfer.log_index + ] + ) + else + from(token_transfer in TokenTransfer, + as: :token_transfer, + join: transaction in assoc(token_transfer, :transaction), + as: :transaction, + join: token in assoc(token_transfer, :token), + as: :token, + join: block in assoc(token_transfer, :block), + as: :block, + select: %TokenTransfer{ + token_transfer + | token_id: fragment("UNNEST(?)", token_transfer.token_ids), + amount: + fragment("UNNEST(COALESCE(?, ARRAY[COALESCE(?, 1)]))", token_transfer.amounts, token_transfer.amount), + reverse_index_in_batch: + fragment("GENERATE_SERIES(COALESCE(ARRAY_LENGTH(?, 1), 1), 1, -1)", token_transfer.amounts), + token_decimals: token.decimals + }, + where: block.consensus == true, + order_by: [ + desc: token_transfer.block_number, + desc: token_transfer.log_index + ] + ) + end + + query_function = + (&make_token_transfer_query_unnested/2) + |> apply_token_transfers_filters(options) + |> page_token_transfers(paging_options) token_transfer_query - |> apply_token_transfers_filters(options) - |> page_token_transfers(paging_options) - |> filter_token_transfers_by_amount(options[:amount][:from], options[:amount][:to]) - |> make_token_transfer_query_unnested() |> limit_query(paging_options) + |> query_function.(false) + |> limit_query(paging_options) + |> preload([:transaction, :token]) + |> select_merge([token_transfer], %{token_ids: [token_transfer.token_id], amounts: [token_transfer.amount]}) + end + + defp page_token_transfers(query_function, %PagingOptions{ + key: %{ + block_number: block_number, + transaction_index: _transaction_index, + token_transfer_index: nil, + internal_transaction_index: nil + } + }) + when block_number < 0 do + fn query, unnested? -> + query |> where(false) |> query_function.(unnested?) + end end - defp page_token_transfers(query, %PagingOptions{ + defp page_token_transfers(query_function, %PagingOptions{ key: %{ block_number: block_number, transaction_index: transaction_index, token_transfer_index: nil, internal_transaction_index: nil } - }) do - case {block_number, transaction_index} do - {0, 0} -> - query |> where(as(:transaction).block_number == ^block_number and as(:transaction).index == ^transaction_index) + }) + when block_number > 0 and transaction_index <= 0 do + fn query, unnested? -> + query |> where([token_transfer], token_transfer.block_number < ^block_number) |> query_function.(unnested?) + end + end - {0, transaction_index} -> - query - |> where( - [token_transfer], - token_transfer.block_number == ^block_number and as(:transaction).index < ^transaction_index - ) + defp page_token_transfers(query_function, %PagingOptions{ + key: %{ + block_number: 0, + transaction_index: 0, + token_transfer_index: nil, + internal_transaction_index: nil + } + }) do + fn query, unnested? -> + query + |> where(as(:transaction).block_number == 0 and as(:transaction).index == 0) + |> query_function.(unnested?) + end + end - {block_number, 0} -> - query |> where([token_transfer], token_transfer.block_number < ^block_number) + defp page_token_transfers(query_function, %PagingOptions{ + key: %{ + block_number: 0, + transaction_index: transaction_index, + token_transfer_index: nil, + internal_transaction_index: nil + } + }) do + fn query, unnested? -> + query + |> where( + [token_transfer], + token_transfer.block_number == 0 and as(:transaction).index < ^transaction_index + ) + |> query_function.(unnested?) + end + end - {block_number, transaction_index} -> - query - |> where( - [token_transfer], - token_transfer.block_number < ^block_number or - (token_transfer.block_number == ^block_number and as(:transaction).index <= ^transaction_index) - ) + defp page_token_transfers(query_function, %PagingOptions{ + key: %{ + block_number: block_number, + transaction_index: transaction_index, + token_transfer_index: nil, + internal_transaction_index: nil + } + }) do + fn query, unnested? -> + query + |> where( + [token_transfer], + token_transfer.block_number < ^block_number or + (token_transfer.block_number == ^block_number and as(:transaction).index <= ^transaction_index) + ) + |> query_function.(unnested?) end end - defp page_token_transfers(query, %PagingOptions{ + defp page_token_transfers(query_function, %PagingOptions{ key: %{ block_number: block_number, transaction_index: transaction_index, @@ -432,10 +579,12 @@ defmodule Explorer.Chain.AdvancedFilter do ^page_transaction_index_dynamic(block_number, transaction_index) ) - query |> where(^dynamic_condition) + fn query, unnested? -> + query |> where(^dynamic_condition) |> query_function.(unnested?) + end end - defp page_token_transfers(query, %PagingOptions{ + defp page_token_transfers(query_function, %PagingOptions{ key: %{ block_number: block_number, token_transfer_index: tt_index, @@ -448,20 +597,21 @@ defmodule Explorer.Chain.AdvancedFilter do ^page_tt_index_dynamic(:token_transfer, block_number, tt_index, tt_batch_index) ) - paged_query = query |> where(^dynamic_condition) - - paged_query - |> make_token_transfer_query_unnested() - |> where( - ^page_tt_batch_index_dynamic( - block_number, - tt_index, - tt_batch_index + fn query, unnested? -> + query + |> where(^dynamic_condition) + |> query_function.(unnested?) + |> where( + ^page_tt_batch_index_dynamic( + block_number, + tt_index, + tt_batch_index + ) ) - ) + end end - defp page_token_transfers(query, _), do: query + defp page_token_transfers(query_function, _), do: query_function defp page_block_number_dynamic(binding, block_number) when block_number > 0 do dynamic(as(^binding).block_number < ^block_number) @@ -471,7 +621,8 @@ defmodule Explorer.Chain.AdvancedFilter do dynamic(false) end - defp page_transaction_index_dynamic(block_number, transaction_index) when transaction_index > 0 do + defp page_transaction_index_dynamic(block_number, transaction_index) + when block_number >= 0 and transaction_index > 0 do dynamic( [transaction: transaction], transaction.block_number == ^block_number and transaction.index < ^transaction_index @@ -482,7 +633,8 @@ defmodule Explorer.Chain.AdvancedFilter do dynamic(false) end - defp page_it_index_dynamic(block_number, transaction_index, it_index) when it_index > 0 do + defp page_it_index_dynamic(block_number, transaction_index, it_index) + when block_number >= 0 and transaction_index >= 0 and it_index > 0 do dynamic( [transaction: transaction, internal_transaction: it], transaction.block_number == ^block_number and transaction.index == ^transaction_index and @@ -495,11 +647,12 @@ defmodule Explorer.Chain.AdvancedFilter do end defp page_tt_index_dynamic(binding, block_number, tt_index, tt_batch_index) - when tt_index > 0 and tt_batch_index > 1 do + when block_number >= 0 and tt_index > 0 and tt_batch_index > 1 do dynamic(as(^binding).block_number == ^block_number and as(^binding).log_index <= ^tt_index) end - defp page_tt_index_dynamic(binding, block_number, tt_index, _tt_batch_index) when tt_index > 0 do + defp page_tt_index_dynamic(binding, block_number, tt_index, _tt_batch_index) + when block_number >= 0 and tt_index > 0 do dynamic(as(^binding).block_number == ^block_number and as(^binding).log_index < ^tt_index) end @@ -507,7 +660,8 @@ defmodule Explorer.Chain.AdvancedFilter do dynamic(false) end - defp page_tt_batch_index_dynamic(block_number, tt_index, tt_batch_index) when tt_batch_index > 1 do + defp page_tt_batch_index_dynamic(block_number, tt_index, tt_batch_index) + when block_number >= 0 and tt_index >= 0 and tt_batch_index > 1 do dynamic( [unnested_token_transfer: tt], ^page_block_number_dynamic(:unnested_token_transfer, block_number) or @@ -529,38 +683,59 @@ defmodule Explorer.Chain.AdvancedFilter do defp limit_query(query, _), do: query - defp apply_token_transfers_filters(query, options) do - query + defp apply_token_transfers_filters(query_function, options) do + query_function |> filter_by_transaction_type(options[:transaction_types]) |> filter_token_transfers_by_methods(options[:methods]) - |> filter_by_token(options[:token_contract_address_hashes][:include], :include) - |> filter_by_token(options[:token_contract_address_hashes][:exclude], :exclude) - |> apply_common_filters(options) + |> filter_token_transfers_by_age(options) + |> filter_by_token(options[:token_contract_address_hashes]) + |> filter_token_transfers_by_addresses( + options[:from_address_hashes], + options[:to_address_hashes], + options[:address_relation] + ) + |> filter_token_transfers_by_amount(options[:amount][:from], options[:amount][:to]) end defp apply_transactions_filters(query, options) do query |> filter_transactions_by_amount(options[:amount][:from], options[:amount][:to]) |> filter_transactions_by_methods(options[:methods]) - |> apply_common_filters(options) + |> only_collated_transactions() + |> filter_by_addresses(options[:from_address_hashes], options[:to_address_hashes], options[:address_relation]) + |> filter_by_age(:transaction, options) end - defp apply_common_filters(query, options) do + defp apply_internal_transactions_filters(query, options) do query + |> filter_transactions_by_amount(options[:amount][:from], options[:amount][:to]) + |> filter_transactions_by_methods(options[:methods]) |> only_collated_transactions() - |> filter_by_timestamp(options[:age][:from], options[:age][:to]) - |> filter_by_addresses(options[:from_address_hashes], options[:to_address_hashes], options[:address_relation]) + |> filter_by_age(:transaction, options) + |> filter_internal_transactions_by_addresses( + options[:from_address_hashes], + options[:to_address_hashes], + options[:address_relation] + ) end defp only_collated_transactions(query) do query |> where(not is_nil(as(:transaction).block_number) and not is_nil(as(:transaction).index)) end - defp filter_by_transaction_type(query, [_ | _] = transaction_types) do - query |> where([token_transfer], token_transfer.token_type in ^transaction_types) + defp filter_by_transaction_type(query_function, [_ | _] = transaction_types) do + if DenormalizationHelper.tt_denormalization_finished?() do + fn query, unnested? -> + query |> where([token_transfer], token_transfer.token_type in ^transaction_types) |> query_function.(unnested?) + end + else + fn query, unnested? -> + query |> where([token: token], token.type in ^transaction_types) |> query_function.(unnested?) + end + end end - defp filter_by_transaction_type(query, _), do: query + defp filter_by_transaction_type(query_function, _), do: query_function defp filter_transactions_by_methods(query, [_ | _] = methods) do prepared_methods = prepare_methods(methods) @@ -570,13 +745,17 @@ defmodule Explorer.Chain.AdvancedFilter do defp filter_transactions_by_methods(query, _), do: query - defp filter_token_transfers_by_methods(query, [_ | _] = methods) do + defp filter_token_transfers_by_methods(query_function, [_ | _] = methods) do prepared_methods = prepare_methods(methods) - query |> where(fragment("substring(? FOR 4)", as(:transaction).input) in ^prepared_methods) + fn query, unnested? -> + query + |> where(fragment("substring(? FOR 4)", as(:transaction).input) in ^prepared_methods) + |> query_function.(unnested?) + end end - defp filter_token_transfers_by_methods(query, _), do: query + defp filter_token_transfers_by_methods(query_function, _), do: query_function defp prepare_methods(methods) do methods @@ -589,16 +768,48 @@ defmodule Explorer.Chain.AdvancedFilter do end) end - defp filter_by_timestamp(query, %DateTime{} = from, %DateTime{} = to) do - query |> where(as(:transaction).block_timestamp >= ^from and as(:transaction).block_timestamp <= ^to) + defp filter_token_transfers_by_age(query_function, options) do + fn query, unnested? -> query |> filter_by_age(:token_transfer, options) |> query_function.(unnested?) end end - defp filter_by_timestamp(query, %DateTime{} = from, _to) do - query |> where(as(:transaction).block_timestamp >= ^from) + defp filter_by_age(query, entity, options) do + query + |> do_filter_by_age(options[:block_numbers_age][:from], options[:age][:from], entity, :from) + |> do_filter_by_age(options[:block_numbers_age][:to], options[:age][:to], entity, :to) + end + + defp do_filter_by_age(query, {:ok, block_number}, _timestamp, entity, direction) do + filter_by_block_number(query, block_number, entity, direction) + end + + defp do_filter_by_age(query, _block_number, timestamp, _entity, direction) do + filter_by_timestamp(query, timestamp, direction) + end + + defp filter_by_block_number(query, from, entity, :from) when not is_nil(from) do + query |> where(as(^entity).block_number >= ^from) + end + + defp filter_by_block_number(query, to, entity, :to) when not is_nil(to) do + query |> where(as(^entity).block_number <= ^to) + end + + defp filter_by_block_number(query, _, _, _), do: query + + defp filter_by_timestamp(query, %DateTime{} = from, :from) do + if DenormalizationHelper.transactions_denormalization_finished?() do + query |> where(as(:transaction).block_timestamp >= ^from) + else + query |> where(as(:block).timestamp >= ^from) + end end - defp filter_by_timestamp(query, _from, %DateTime{} = to) do - query |> where(as(:transaction).block_timestamp <= ^to) + defp filter_by_timestamp(query, %DateTime{} = to, :to) do + if DenormalizationHelper.transactions_denormalization_finished?() do + query |> where(as(:transaction).block_timestamp <= ^to) + else + query |> where(as(:block).timestamp <= ^to) + end end defp filter_by_timestamp(query, _, _), do: query @@ -656,84 +867,566 @@ defmodule Explorer.Chain.AdvancedFilter do dynamic([t], ^from_addresses_dynamic and ^to_addresses_dynamic) end - @eth_decimals 1000_000_000_000_000_000 + defp filter_token_transfers_by_addresses(query_function, from_addresses_params, to_addresses_params, relation) do + case {process_address_inclusion(from_addresses_params), process_address_inclusion(to_addresses_params)} do + {nil, nil} -> query_function + {from, nil} -> do_filter_token_transfers_by_address(query_function, from, :from_address_hash) + {nil, to} -> do_filter_token_transfers_by_address(query_function, to, :to_address_hash) + {from, to} -> do_filter_token_transfers_by_both_addresses(query_function, from, to, relation) + end + end + + defp do_filter_token_transfers_by_address(query_function, {:include, addresses}, field) do + fn query, _unnested? -> + queries = + addresses + |> Enum.map(fn address -> + query |> where([token_transfer], field(token_transfer, ^field) == ^address) |> query_function.(true) + end) + |> map_first(&subquery/1) + |> Enum.reduce(fn query, acc -> union_all(acc, ^query) end) + + from(token_transfer in subquery(queries), + as: :unnested_token_transfer, + order_by: [desc: token_transfer.block_number, desc: token_transfer.log_index] + ) + end + end + + defp do_filter_token_transfers_by_address(query_function, {:exclude, addresses}, field) do + fn query, unnested? -> + query |> where([t], field(t, ^field) not in ^addresses) |> query_function.(unnested?) + end + end - defp filter_transactions_by_amount(query, from, to) when not is_nil(from) and not is_nil(to) and from < to do - query |> where([t], t.value / @eth_decimals >= ^from and t.value / @eth_decimals <= ^to) + defp do_filter_token_transfers_by_both_addresses(query_function, {:include, from}, {:include, to}, relation) do + fn query, _unnested? -> + from_queries = + from + |> Enum.map(fn from_address -> + query |> where([token_transfer], token_transfer.from_address_hash == ^from_address) |> query_function.(true) + end) + + to_queries = + to + |> Enum.map(fn to_address -> + query |> where([token_transfer], token_transfer.to_address_hash == ^to_address) |> query_function.(true) + end) + + do_filter_token_transfers_by_both_addresses_to_include(from_queries, to_queries, relation) + end end - defp filter_transactions_by_amount(query, _from, to) when not is_nil(to) do - query |> where([t], t.value / @eth_decimals <= ^to) + defp do_filter_token_transfers_by_both_addresses(query_function, {:include, from}, {:exclude, to}, :and) do + fn query, _unnested? -> + from_queries = + from + |> Enum.map(fn from_address -> + query + |> where( + [token_transfer], + token_transfer.from_address_hash == ^from_address and token_transfer.to_address_hash not in ^to + ) + |> query_function.(true) + end) + |> map_first(&subquery/1) + |> Enum.reduce(fn query, acc -> union_all(acc, ^query) end) + + from(token_transfer in subquery(from_queries), + as: :unnested_token_transfer, + order_by: [desc: token_transfer.block_number, desc: token_transfer.log_index] + ) + end end - defp filter_transactions_by_amount(query, from, _to) when not is_nil(from) do - query |> where([t], t.value / @eth_decimals >= ^from) + defp do_filter_token_transfers_by_both_addresses(query_function, {:include, from}, {:exclude, to}, _relation) do + fn query, _unnested? -> + from_queries = + from + |> Enum.map(fn from_address -> + query + |> where( + [token_transfer], + token_transfer.from_address_hash == ^from_address or token_transfer.to_address_hash not in ^to + ) + |> query_function.(true) + end) + |> map_first(&subquery/1) + |> Enum.reduce(fn query, acc -> union_all(acc, ^query) end) + + from(token_transfer in subquery(from_queries), + as: :unnested_token_transfer, + order_by: [desc: token_transfer.block_number, desc: token_transfer.log_index] + ) + end end - defp filter_transactions_by_amount(query, _, _), do: query + defp do_filter_token_transfers_by_both_addresses(query_function, {:exclude, from}, {:include, to}, :and) do + fn query, _unnested? -> + to_queries = + to + |> Enum.map(fn to_address -> + query + |> where( + [token_transfer], + token_transfer.to_address_hash == ^to_address and token_transfer.from_address_hash not in ^from + ) + |> query_function.(true) + end) + |> map_first(&subquery/1) + |> Enum.reduce(fn query, acc -> union_all(acc, ^query) end) + + from(token_transfer in subquery(to_queries), + as: :unnested_token_transfer, + order_by: [desc: token_transfer.block_number, desc: token_transfer.log_index] + ) + end + end - defp filter_token_transfers_by_amount(query, from, to) when not is_nil(from) and not is_nil(to) and from < to do - unnested_query = make_token_transfer_query_unnested(query) + defp do_filter_token_transfers_by_both_addresses(query_function, {:exclude, from}, {:include, to}, _relation) do + fn query, _unnested? -> + to_queries = + to + |> Enum.map(fn to_address -> + query + |> where( + [token_transfer], + token_transfer.to_address_hash == ^to_address or token_transfer.from_address_hash not in ^from + ) + |> query_function.(true) + end) + |> map_first(&subquery/1) + |> Enum.reduce(fn query, acc -> union_all(acc, ^query) end) + + from(token_transfer in subquery(to_queries), + as: :unnested_token_transfer, + order_by: [desc: token_transfer.block_number, desc: token_transfer.log_index] + ) + end + end - unnested_query - |> where( - [unnested_token_transfer: tt], - tt.amount / fragment("10 ^ COALESCE(?, 0)", tt.token_decimals) >= ^from and - tt.amount / fragment("10 ^ COALESCE(?, 0)", tt.token_decimals) <= ^to + defp do_filter_token_transfers_by_both_addresses(query_function, {:exclude, from}, {:exclude, to}, :and) do + fn query, unnested? -> + query + |> where([t], t.from_address_hash not in ^from and t.to_address_hash not in ^to) + |> query_function.(unnested?) + end + end + + defp do_filter_token_transfers_by_both_addresses(query_function, {:exclude, from}, {:exclude, to}, _relation) do + fn query, unnested? -> + query + |> where([t], t.from_address_hash not in ^from or t.to_address_hash not in ^to) + |> query_function.(unnested?) + end + end + + defp do_filter_token_transfers_by_both_addresses_to_include(from_queries, to_queries, relation) do + case relation do + :and -> + united_from_queries = + from_queries |> map_first(&subquery/1) |> Enum.reduce(fn query, acc -> union_all(acc, ^query) end) + + united_to_queries = + to_queries |> map_first(&subquery/1) |> Enum.reduce(fn query, acc -> union_all(acc, ^query) end) + + from(token_transfer in subquery(intersect_all(united_from_queries, ^united_to_queries)), + as: :unnested_token_transfer, + order_by: [desc: token_transfer.block_number, desc: token_transfer.log_index] + ) + + _ -> + union_query = + from_queries + |> Kernel.++(to_queries) + |> map_first(&subquery/1) + |> Enum.reduce(fn query, acc -> union(acc, ^query) end) + + from(token_transfer in subquery(union_query), + as: :unnested_token_transfer, + order_by: [desc: token_transfer.block_number, desc: token_transfer.log_index] + ) + end + end + + defp filter_internal_transactions_by_addresses(query, from_addresses, to_addresses, relation) do + case {process_address_inclusion(from_addresses), process_address_inclusion(to_addresses)} do + {nil, nil} -> query + {from, nil} -> do_filter_internal_transactions_by_address(query, from, :from_address_hash) + {nil, to} -> do_filter_internal_transactions_by_address(query, to, :to_address_hash) + {from, to} -> do_filter_internal_transactions_by_both_addresses(query, from, to, relation) + end + end + + defp do_filter_internal_transactions_by_address(query, {:include, addresses}, field) do + queries = + addresses + |> Enum.map(fn address -> + query |> where([token_transfer], field(token_transfer, ^field) == ^address) + end) + |> map_first(&subquery/1) + |> Enum.reduce(fn query, acc -> union_all(acc, ^query) end) + + from(internal_transaction in subquery(queries), + order_by: [ + desc: internal_transaction.block_number, + desc: internal_transaction.transaction_index, + desc: internal_transaction.index + ] ) end - defp filter_token_transfers_by_amount(query, _from, to) when not is_nil(to) do - unnested_query = make_token_transfer_query_unnested(query) + defp do_filter_internal_transactions_by_address(query, {:exclude, addresses}, field) do + query |> where([t], field(t, ^field) not in ^addresses) + end - unnested_query - |> where( - [unnested_token_transfer: tt], - tt.amount / fragment("10 ^ COALESCE(?, 0)", tt.token_decimals) <= ^to + defp do_filter_internal_transactions_by_both_addresses(query, {:include, from}, {:include, to}, relation) do + from_queries = + from + |> Enum.map(fn from_address -> + query |> where([internal_transaction], internal_transaction.from_address_hash == ^from_address) + end) + + to_queries = + to + |> Enum.map(fn to_address -> + query |> where([internal_transaction], internal_transaction.to_address_hash == ^to_address) + end) + + do_filter_internal_transactions_by_both_addresses_to_include(from_queries, to_queries, relation) + end + + defp do_filter_internal_transactions_by_both_addresses(query, {:include, from}, {:exclude, to}, :and) do + from_queries = + from + |> Enum.map(fn from_address -> + query + |> where( + [internal_transaction], + internal_transaction.from_address_hash == ^from_address and internal_transaction.to_address_hash not in ^to + ) + end) + |> map_first(&subquery/1) + |> Enum.reduce(fn query, acc -> union_all(acc, ^query) end) + + from(internal_transaction in subquery(from_queries), + order_by: [ + desc: internal_transaction.block_number, + desc: internal_transaction.transaction_index, + desc: internal_transaction.index + ] + ) + end + + defp do_filter_internal_transactions_by_both_addresses(query, {:include, from}, {:exclude, to}, _relation) do + from_queries = + from + |> Enum.map(fn from_address -> + query + |> where( + [internal_transaction], + internal_transaction.from_address_hash == ^from_address or internal_transaction.to_address_hash not in ^to + ) + end) + |> map_first(&subquery/1) + |> Enum.reduce(fn query, acc -> union_all(acc, ^query) end) + + from(internal_transaction in subquery(from_queries), + order_by: [ + desc: internal_transaction.block_number, + desc: internal_transaction.transaction_index, + desc: internal_transaction.index + ] ) end - defp filter_token_transfers_by_amount(query, from, _to) when not is_nil(from) do - unnested_query = make_token_transfer_query_unnested(query) + defp do_filter_internal_transactions_by_both_addresses(query, {:exclude, from}, {:include, to}, :and) do + to_queries = + to + |> Enum.map(fn to_address -> + query + |> where( + [internal_transaction], + internal_transaction.to_address_hash == ^to_address and internal_transaction.from_address_hash not in ^from + ) + end) + |> map_first(&subquery/1) + |> Enum.reduce(fn query, acc -> union_all(acc, ^query) end) + + from(internal_transaction in subquery(to_queries), + order_by: [ + desc: internal_transaction.block_number, + desc: internal_transaction.transaction_index, + desc: internal_transaction.index + ] + ) + end - unnested_query - |> where( - [unnested_token_transfer: tt], - tt.amount / fragment("10 ^ COALESCE(?, 0)", tt.token_decimals) >= ^from + defp do_filter_internal_transactions_by_both_addresses(query, {:exclude, from}, {:include, to}, _relation) do + to_queries = + to + |> Enum.map(fn to_address -> + query + |> where( + [internal_transaction], + internal_transaction.to_address_hash == ^to_address or internal_transaction.from_address_hash not in ^from + ) + end) + |> map_first(&subquery/1) + |> Enum.reduce(fn query, acc -> union_all(acc, ^query) end) + + from(internal_transaction in subquery(to_queries), + order_by: [ + desc: internal_transaction.block_number, + desc: internal_transaction.transaction_index, + desc: internal_transaction.index + ] ) end - defp filter_token_transfers_by_amount(query, _, _), do: query + defp do_filter_internal_transactions_by_both_addresses(query, {:exclude, from}, {:exclude, to}, :and) do + query + |> where([t], t.from_address_hash not in ^from and t.to_address_hash not in ^to) + end + + defp do_filter_internal_transactions_by_both_addresses(query, {:exclude, from}, {:exclude, to}, _relation) do + query + |> where([t], t.from_address_hash not in ^from or t.to_address_hash not in ^to) + end + + defp do_filter_internal_transactions_by_both_addresses_to_include(from_queries, to_queries, relation) do + case relation do + :and -> + united_from_queries = + from_queries |> map_first(&subquery/1) |> Enum.reduce(fn query, acc -> union_all(acc, ^query) end) + + united_to_queries = + to_queries |> map_first(&subquery/1) |> Enum.reduce(fn query, acc -> union_all(acc, ^query) end) + + from(internal_transaction in subquery(intersect_all(united_from_queries, ^united_to_queries)), + order_by: [ + desc: internal_transaction.block_number, + desc: internal_transaction.transaction_index, + desc: internal_transaction.index + ] + ) + + _ -> + union_query = + from_queries + |> Kernel.++(to_queries) + |> map_first(&subquery/1) + |> Enum.reduce(fn query, acc -> union(acc, ^query) end) + + from(internal_transaction in subquery(union_query), + order_by: [ + desc: internal_transaction.block_number, + desc: internal_transaction.transaction_index, + desc: internal_transaction.index + ] + ) + end + end + + @eth_decimals 1_000_000_000_000_000_000 + + defp filter_transactions_by_amount(query, from, to) when not is_nil(from) and not is_nil(to) do + if Decimal.positive?(to) and Decimal.lt?(from, to) do + query |> where([t], t.value / @eth_decimals >= ^from and t.value / @eth_decimals <= ^to) + else + query |> where(false) + end + end + + defp filter_transactions_by_amount(query, _from, to) when not is_nil(to) do + if Decimal.positive?(to) do + query |> where([t], t.value / @eth_decimals <= ^to) + else + query |> where(false) + end + end + + defp filter_transactions_by_amount(query, from, _to) when not is_nil(from) do + if Decimal.positive?(from) do + query |> where([t], t.value / @eth_decimals >= ^from) + else + query + end + end + + defp filter_transactions_by_amount(query, _, _), do: query + + defp filter_token_transfers_by_amount(query_function, from, to) do + fn query, unnested? -> + query + |> filter_token_transfers_by_amount_before_subquery(from, to) + |> query_function.(unnested?) + |> filter_token_transfers_by_amount_after_subquery(from, to) + end + end + + defp filter_token_transfers_by_amount_before_subquery(query, from, to) + when not is_nil(from) and not is_nil(to) and from < to do + if Decimal.positive?(to) and Decimal.lt?(from, to) do + query + |> where( + [tt, token: token], + ^to * fragment("10 ^ COALESCE(?, 0)", token.decimals) >= + fragment("ANY(COALESCE(?, ARRAY[COALESCE(?, 1)]))", tt.amounts, tt.amount) and + ^from * fragment("10 ^ COALESCE(?, 0)", token.decimals) <= + fragment("ANY(COALESCE(?, ARRAY[COALESCE(?, 1)]))", tt.amounts, tt.amount) + ) + else + query |> where(false) + end + end + + defp filter_token_transfers_by_amount_before_subquery(query, _from, to) when not is_nil(to) do + if Decimal.positive?(to) do + query + |> where( + [tt, token: token], + ^to * fragment("10 ^ COALESCE(?, 0)", token.decimals) >= + fragment("ANY(COALESCE(?, ARRAY[COALESCE(?, 1)]))", tt.amounts, tt.amount) + ) + else + query |> where(false) + end + end - defp make_token_transfer_query_unnested(query) do - if has_named_binding?(query, :unnested_token_transfer) do + defp filter_token_transfers_by_amount_before_subquery(query, from, _to) when not is_nil(from) do + if Decimal.positive?(from) do + query + |> where( + [tt, token: token], + ^from * fragment("10 ^ COALESCE(?, 0)", token.decimals) <= + fragment("ANY(COALESCE(?, ARRAY[COALESCE(?, 1)]))", tt.amounts, tt.amount) + ) + else query + end + end + + defp filter_token_transfers_by_amount_before_subquery(query, _, _), do: query + + defp filter_token_transfers_by_amount_after_subquery(unnested_query, from, to) + when not is_nil(from) and not is_nil(to) and from < to do + if Decimal.positive?(to) and Decimal.lt?(from, to) do + unnested_query + |> where( + [unnested_token_transfer: tt], + tt.amount / fragment("10 ^ COALESCE(?, 0)", tt.token_decimals) >= ^from and + tt.amount / fragment("10 ^ COALESCE(?, 0)", tt.token_decimals) <= ^to + ) + else + unnested_query |> where(false) + end + end + + defp filter_token_transfers_by_amount_after_subquery(unnested_query, _from, to) when not is_nil(to) do + if Decimal.positive?(to) do + unnested_query + |> where( + [unnested_token_transfer: tt], + tt.amount / fragment("10 ^ COALESCE(?, 0)", tt.token_decimals) <= ^to + ) + else + unnested_query |> where(false) + end + end + + defp filter_token_transfers_by_amount_after_subquery(unnested_query, from, _to) when not is_nil(from) do + if Decimal.positive?(from) do + unnested_query + |> where( + [unnested_token_transfer: tt], + tt.amount / fragment("10 ^ COALESCE(?, 0)", tt.token_decimals) >= ^from + ) else + unnested_query + end + end + + defp filter_token_transfers_by_amount_after_subquery(query, _, _), do: query + + defp make_token_transfer_query_unnested(query, false) do + with_named_binding(query, :unnested_token_transfer, fn query, binding -> from(token_transfer in subquery(query), + as: ^binding + ) + end) + end + + defp make_token_transfer_query_unnested(query, _), do: query + + defp filter_by_token(query_function, token_contract_address_hashes) when is_list(token_contract_address_hashes) do + case process_address_inclusion(token_contract_address_hashes) do + nil -> + query_function + + {include_or_exclude, token_contract_address_hashes} -> + filtered = token_contract_address_hashes |> Enum.reject(&(&1 == "native")) + + if Enum.empty?(filtered) do + query_function + else + do_filter_by_token(query_function, {include_or_exclude, filtered}) + end + end + end + + defp filter_by_token(query_function, _), do: query_function + + defp do_filter_by_token(query_function, {:include, token_contract_address_hashes}) do + fn query, _unnested? -> + queries = + token_contract_address_hashes + |> Enum.map(fn address -> + query + |> where([token_transfer], token_transfer.token_contract_address_hash == ^address) + |> query_function.(true) + end) + |> map_first(&subquery/1) + |> Enum.reduce(fn query, acc -> union_all(acc, ^query) end) + + from(token_transfer in subquery(queries), as: :unnested_token_transfer, - preload: [ - :transaction, - :token, - from_address: [:scam_badge, :names, :smart_contract, :proxy_implementations], - to_address: [:scam_badge, :names, :smart_contract, :proxy_implementations] - ], - select_merge: %{ - token_ids: [token_transfer.token_id], - amounts: [token_transfer.amount] - } + order_by: [desc: token_transfer.block_number, desc: token_transfer.log_index] ) end end - defp filter_by_token(query, [_ | _] = token_contract_address_hashes, :include) do - filtered = token_contract_address_hashes |> Enum.reject(&(&1 == "native")) - query |> where([token_transfer], token_transfer.token_contract_address_hash in ^filtered) + defp do_filter_by_token(query_function, {:exclude, token_contract_address_hashes}) do + fn query, unnested? -> + query_function.( + from(token_transfer in query, + left_join: to_exclude in fragment("UNNEST(?)", type(^token_contract_address_hashes, {:array, Hash.Address})), + on: token_transfer.token_contract_address_hash == to_exclude, + where: is_nil(to_exclude) + ), + unnested? + ) + end end - defp filter_by_token(query, [_ | _] = token_contract_address_hashes, :exclude) do - filtered = token_contract_address_hashes |> Enum.reject(&(&1 == "native")) - query |> where([token_transfer], token_transfer.token_contract_address_hash not in ^filtered) + defp process_address_inclusion(addresses) when is_list(addresses) do + case {Keyword.get(addresses, :include, []), Keyword.get(addresses, :exclude, [])} do + {to_include, to_exclude} when to_include in [nil, []] and to_exclude in [nil, []] -> + nil + + {to_include, to_exclude} when to_include in [nil, []] and is_list(to_exclude) -> + {:exclude, to_exclude} + + {to_include, to_exclude} when is_list(to_include) -> + case to_include -- (to_exclude || []) do + [] -> nil + to_include -> {:include, to_include} + end + end end - defp filter_by_token(query, _, _), do: query + defp process_address_inclusion(_), do: nil + + defp map_first([h | t], f), do: [f.(h) | t] + defp map_first([], _f), do: [] end