diff --git a/CHANGELOG.md b/CHANGELOG.md index dc9d1aaa0f..d56ddd37a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ## Current ### Features +- [#2283](https://github.com/poanetwork/blockscout/pull/2283) - Add transactions cache - [#2182](https://github.com/poanetwork/blockscout/pull/2182) - add market history cache - [#2109](https://github.com/poanetwork/blockscout/pull/2109) - use bigger updates instead of `Multi` transactions in BlocksTransactionsMismatch - [#2075](https://github.com/poanetwork/blockscout/pull/2075) - add blocks cache @@ -13,6 +14,8 @@ - [#2266](https://github.com/poanetwork/blockscout/pull/2266) - allow excluding uncles from average block time calculation ### Fixes +- [#2284](https://github.com/poanetwork/blockscout/pull/2284) - add 404 status for not existing pages +- [#2244](https://github.com/poanetwork/blockscout/pull/2244) - fix internal transactions failing to be indexed because of constraint - [#2281](https://github.com/poanetwork/blockscout/pull/2281) - typo issues, dropdown issues - [#2278](https://github.com/poanetwork/blockscout/pull/2278) - increase threshold for scientific notation - [#2275](https://github.com/poanetwork/blockscout/pull/2275) - Description for networks selector @@ -62,8 +65,11 @@ - [#2167](https://github.com/poanetwork/blockscout/pull/2167) - feat: document eth rpc api mimicking endpoints - [#2225](https://github.com/poanetwork/blockscout/pull/2225) - fix metadata decoding in Solidity 0.5.9 smart contract verification - [#2204](https://github.com/poanetwork/blockscout/pull/2204) - fix large contract verification +- [#2258](https://github.com/poanetwork/blockscout/pull/2258) - reduce BlocksTransactionsMismatch memory footprint - [#2247](https://github.com/poanetwork/blockscout/pull/2247) - hide logs search if there are no logs - [#2248](https://github.com/poanetwork/blockscout/pull/2248) - sort block after query execution for average block time +- [#2249](https://github.com/poanetwork/blockscout/pull/2249) - More transaction controllers improvements +- [#2267](https://github.com/poanetwork/blockscout/pull/2267) - Modify implementation of `where_transaction_has_multiple_internal_transactions` - [#2270](https://github.com/poanetwork/blockscout/pull/2270) - Remove duplicate params in `Indexer.Fetcher.TokenBalance` - [#2268](https://github.com/poanetwork/blockscout/pull/2268) - remove not existing assigns in html code - [#2276](https://github.com/poanetwork/blockscout/pull/2276) - remove port in docs diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/block_transaction_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/block_transaction_controller.ex index f54671e795..15b06f1edd 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/block_transaction_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/block_transaction_controller.ex @@ -17,7 +17,7 @@ defmodule BlockScoutWeb.BlockTransactionController do Keyword.merge( [ necessity_by_association: %{ - :block => :required, + :block => :optional, [created_contract_address: :names] => :optional, [from_address: :names] => :required, [to_address: :names] => :optional diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/page_not_found_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/page_not_found_controller.ex index e453e5463b..7ea8fab254 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/page_not_found_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/page_not_found_controller.ex @@ -3,6 +3,7 @@ defmodule BlockScoutWeb.PageNotFoundController do def index(conn, _params) do conn + |> put_status(:not_found) |> render("index.html") end end diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/recent_transactions_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/recent_transactions_controller.ex index 4f66597bcb..b001093c6e 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/recent_transactions_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/recent_transactions_controller.ex @@ -12,7 +12,7 @@ defmodule BlockScoutWeb.RecentTransactionsController do necessity_by_association: %{ :block => :required, [created_contract_address: :names] => :optional, - [from_address: :names] => :required, + [from_address: :names] => :optional, [to_address: :names] => :optional }, paging_options: %PagingOptions{page_size: 5} diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/transaction_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/transaction_controller.ex index b40e6988e0..1bedf97c1d 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/transaction_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/transaction_controller.ex @@ -61,21 +61,16 @@ defmodule BlockScoutWeb.TransactionController do end def show(conn, %{"id" => id}) do - case Chain.string_to_transaction_hash(id) do - {:ok, transaction_hash} -> show_transaction(conn, id, Chain.hash_to_transaction(transaction_hash)) - :error -> conn |> put_status(422) |> render("invalid.html", transaction_hash: id) - end - end - - defp show_transaction(conn, id, {:error, :not_found}) do - conn |> put_status(404) |> render("not_found.html", transaction_hash: id) - end - - defp show_transaction(conn, id, {:ok, %Chain.Transaction{} = transaction}) do - if Chain.transaction_has_token_transfers?(transaction.hash) do - redirect(conn, to: transaction_token_transfer_path(conn, :index, id)) + with {:ok, transaction_hash} <- Chain.string_to_transaction_hash(id), + {:ok, %Chain.Transaction{} = transaction} <- Chain.hash_to_transaction(transaction_hash) do + if Chain.transaction_has_token_transfers?(transaction.hash) do + redirect(conn, to: transaction_token_transfer_path(conn, :index, id)) + else + redirect(conn, to: transaction_internal_transaction_path(conn, :index, id)) + end else - redirect(conn, to: transaction_internal_transaction_path(conn, :index, id)) + :error -> conn |> put_status(422) |> render("invalid.html", transaction_hash: id) + {:error, :not_found} -> conn |> put_status(404) |> render("not_found.html", transaction_hash: id) end end end diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/transaction_internal_transaction_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/transaction_internal_transaction_controller.ex index 7c24b8d3b9..159d144cdd 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/transaction_internal_transaction_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/transaction_internal_transaction_controller.ex @@ -17,7 +17,8 @@ defmodule BlockScoutWeb.TransactionInternalTransactionController do necessity_by_association: %{ [created_contract_address: :names] => :optional, [from_address: :names] => :optional, - [to_address: :names] => :optional + [to_address: :names] => :optional, + [transaction: :block] => :optional } ], paging_options(params) diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/transaction_raw_trace_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/transaction_raw_trace_controller.ex index a73290faab..3d090e8a9b 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/transaction_raw_trace_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/transaction_raw_trace_controller.ex @@ -19,15 +19,7 @@ defmodule BlockScoutWeb.TransactionRawTraceController do :token_transfers => :optional } ) do - options = [ - necessity_by_association: %{ - [created_contract_address: :names] => :optional, - [from_address: :names] => :optional, - [to_address: :names] => :optional - } - ] - - internal_transactions = Chain.transaction_to_internal_transactions(transaction, options) + internal_transactions = Chain.transaction_to_internal_transactions(transaction) render( conn, diff --git a/apps/block_scout_web/test/block_scout_web/controllers/page_not_found_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/page_not_found_controller_test.exs new file mode 100644 index 0000000000..85403586ee --- /dev/null +++ b/apps/block_scout_web/test/block_scout_web/controllers/page_not_found_controller_test.exs @@ -0,0 +1,11 @@ +defmodule BlockScoutWeb.PageNotFoundControllerTest do + use BlockScoutWeb.ConnCase + + describe "GET index/2" do + test "returns 404 status", %{conn: conn} do + conn = get(conn, "/wrong", %{}) + + assert html_response(conn, 404) + end + end +end diff --git a/apps/block_scout_web/test/support/conn_case.ex b/apps/block_scout_web/test/support/conn_case.ex index 6c3f24d89a..d0b9b066a1 100644 --- a/apps/block_scout_web/test/support/conn_case.ex +++ b/apps/block_scout_web/test/support/conn_case.ex @@ -38,6 +38,9 @@ defmodule BlockScoutWeb.ConnCase do Ecto.Adapters.SQL.Sandbox.mode(Explorer.Repo, {:shared, self()}) end + Supervisor.terminate_child(Explorer.Supervisor, {ConCache, Explorer.Chain.TransactionsCache.cache_name()}) + Supervisor.restart_child(Explorer.Supervisor, {ConCache, Explorer.Chain.TransactionsCache.cache_name()}) + {:ok, conn: Phoenix.ConnTest.build_conn()} end end diff --git a/apps/block_scout_web/test/support/feature_case.ex b/apps/block_scout_web/test/support/feature_case.ex index cf9d64cdc5..b9e6f000d7 100644 --- a/apps/block_scout_web/test/support/feature_case.ex +++ b/apps/block_scout_web/test/support/feature_case.ex @@ -27,6 +27,9 @@ defmodule BlockScoutWeb.FeatureCase do Ecto.Adapters.SQL.Sandbox.mode(Explorer.Repo, {:shared, self()}) end + Supervisor.terminate_child(Explorer.Supervisor, {ConCache, Explorer.Chain.TransactionsCache.cache_name()}) + Supervisor.restart_child(Explorer.Supervisor, {ConCache, Explorer.Chain.TransactionsCache.cache_name()}) + metadata = Phoenix.Ecto.SQL.Sandbox.metadata_for(Explorer.Repo, self()) {:ok, session} = Wallaby.start_session(metadata: metadata) session = Wallaby.Browser.resize_window(session, 1200, 800) diff --git a/apps/explorer/lib/explorer/application.ex b/apps/explorer/lib/explorer/application.ex index 724e4f41a1..bb1108f17d 100644 --- a/apps/explorer/lib/explorer/application.ex +++ b/apps/explorer/lib/explorer/application.ex @@ -6,7 +6,16 @@ defmodule Explorer.Application do use Application alias Explorer.Admin - alias Explorer.Chain.{BlockCountCache, BlockNumberCache, BlocksCache, NetVersionCache, TransactionCountCache} + + alias Explorer.Chain.{ + BlockCountCache, + BlockNumberCache, + BlocksCache, + NetVersionCache, + TransactionCountCache, + TransactionsCache + } + alias Explorer.Market.MarketHistoryCache alias Explorer.Repo.PrometheusLogger @@ -34,7 +43,8 @@ defmodule Explorer.Application do {BlockCountCache, []}, con_cache_child_spec(BlocksCache.cache_name()), con_cache_child_spec(NetVersionCache.cache_name()), - con_cache_child_spec(MarketHistoryCache.cache_name()) + con_cache_child_spec(MarketHistoryCache.cache_name()), + con_cache_child_spec(TransactionsCache.cache_name()) ] children = base_children ++ configurable_children() diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index cb2c67ef55..62ff70afb8 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -46,6 +46,7 @@ defmodule Explorer.Chain do TokenTransfer, Transaction, TransactionCountCache, + TransactionsCache, Wei } @@ -1052,7 +1053,7 @@ defmodule Explorer.Chain do when is_list(options) do necessity_by_association = Keyword.get(options, :necessity_by_association, %{}) - fetch_transactions() + Transaction |> where(hash: ^hash) |> join_associations(necessity_by_association) |> Repo.one() @@ -1948,12 +1949,29 @@ defmodule Explorer.Chain do @spec recent_collated_transactions([paging_options | necessity_by_association_option]) :: [Transaction.t()] def recent_collated_transactions(options \\ []) when is_list(options) do necessity_by_association = Keyword.get(options, :necessity_by_association, %{}) + paging_options = Keyword.get(options, :paging_options, @default_paging_options) - options - |> Keyword.get(:paging_options, @default_paging_options) + if is_nil(paging_options.key) do + paging_options.page_size + |> TransactionsCache.take_enough() + |> case do + nil -> + transactions = fetch_recent_collated_transactions(paging_options, necessity_by_association) + TransactionsCache.update(transactions) + transactions + + transactions -> + transactions + end + else + fetch_recent_collated_transactions(paging_options, necessity_by_association) + end + end + + def fetch_recent_collated_transactions(paging_options, necessity_by_association) do + paging_options |> fetch_transactions() |> where([transaction], not is_nil(transaction.block_number) and not is_nil(transaction.index)) - |> order_by([transaction], desc: transaction.block_number, desc: transaction.index) |> join_associations(necessity_by_association) |> preload([{:token_transfers, [:token, :from_address, :to_address]}]) |> Repo.all() @@ -2146,7 +2164,7 @@ defmodule Explorer.Chain do |> page_internal_transaction(paging_options) |> limit(^paging_options.page_size) |> order_by([internal_transaction], asc: internal_transaction.index) - |> preload(transaction: :block) + |> preload(:transaction) |> Repo.all() end @@ -2580,14 +2598,14 @@ defmodule Explorer.Chain do internal_transaction.type != ^:call or fragment( """ - (SELECT COUNT(sibling.*) + EXISTS (SELECT sibling.* FROM internal_transactions AS sibling - WHERE sibling.transaction_hash = ? - LIMIT 2 + WHERE sibling.transaction_hash = ? AND sibling.index != ? ) """, - transaction.hash - ) > 1 + transaction.hash, + internal_transaction.index + ) ) end @@ -2707,9 +2725,9 @@ defmodule Explorer.Chain do @spec transaction_has_token_transfers?(Hash.t()) :: boolean() def transaction_has_token_transfers?(transaction_hash) do - query = from(tt in TokenTransfer, where: tt.transaction_hash == ^transaction_hash, limit: 1, select: 1) + query = from(tt in TokenTransfer, where: tt.transaction_hash == ^transaction_hash) - Repo.one(query) != nil + Repo.exists?(query) end @spec address_tokens_with_balance(Hash.Address.t(), [any()]) :: [] diff --git a/apps/explorer/lib/explorer/chain/import/runner/internal_transactions.ex b/apps/explorer/lib/explorer/chain/import/runner/internal_transactions.ex index 45bd9e3f43..5e2e4e32f5 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/internal_transactions.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/internal_transactions.ex @@ -171,8 +171,7 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactions do ), status: fragment( - "COALESCE(?, CASE WHEN (SELECT it.error FROM internal_transactions AS it WHERE it.transaction_hash = ? ORDER BY it.index ASC LIMIT 1) IS NULL THEN ? ELSE ? END)", - t.status, + "CASE WHEN (SELECT it.error FROM internal_transactions AS it WHERE it.transaction_hash = ? ORDER BY it.index ASC LIMIT 1) IS NULL THEN ? ELSE ? END", t.hash, type(^:ok, t.status), type(^:error, t.status) diff --git a/apps/explorer/lib/explorer/chain/import/runner/transactions.ex b/apps/explorer/lib/explorer/chain/import/runner/transactions.ex index 1a9fcc497d..2c36d7c7e5 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/transactions.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/transactions.ex @@ -90,8 +90,7 @@ defmodule Explorer.Chain.Import.Runner.Transactions do conflict_target: :hash, on_conflict: on_conflict, for: Transaction, - returning: - ~w(block_number index hash internal_transactions_indexed_at block_hash old_block_hash nonce from_address_hash created_contract_address_hash)a, + returning: true, timeout: timeout, timestamps: timestamps ) diff --git a/apps/explorer/lib/explorer/chain/transactions_cache.ex b/apps/explorer/lib/explorer/chain/transactions_cache.ex new file mode 100644 index 0000000000..3859561295 --- /dev/null +++ b/apps/explorer/lib/explorer/chain/transactions_cache.ex @@ -0,0 +1,143 @@ +defmodule Explorer.Chain.TransactionsCache do + @moduledoc """ + Caches the latest imported transactions + """ + + alias Explorer.Chain.Transaction + alias Explorer.Repo + + @transactions_ids_key "transactions_ids" + @cache_name :transactions + @max_size 51 + @preloads [ + :block, + created_contract_address: :names, + from_address: :names, + to_address: :names, + token_transfers: :token, + token_transfers: :from_address, + token_transfers: :to_address + ] + + @spec cache_name :: atom() + def cache_name, do: @cache_name + + @doc """ + Fetches a transaction from its id ({block_number, index}), returns nil if not found + """ + @spec get({non_neg_integer(), non_neg_integer()}) :: Transaction.t() | nil + def get(id), do: ConCache.get(@cache_name, id) + + @doc """ + Return the current number of transactions stored + """ + @spec size :: non_neg_integer() + def size, do: Enum.count(transactions_ids()) + + @doc """ + Checks if there are enough transactions stored + """ + @spec enough?(non_neg_integer()) :: boolean() + def enough?(amount) do + amount <= size() + end + + @doc """ + Checks if the number of transactions stored is already the max allowed + """ + @spec full? :: boolean() + def full? do + @max_size <= size() + end + + @doc "Returns the list ids of the transactions currently stored" + @spec transactions_ids :: [{non_neg_integer(), non_neg_integer()}] + def transactions_ids do + ConCache.get(@cache_name, @transactions_ids_key) || [] + end + + @doc "Returns all the stored transactions" + @spec all :: [Transaction.t()] + def all, do: Enum.map(transactions_ids(), &get(&1)) + + @doc "Returns the `n` most recent transactions stored" + @spec take(integer()) :: [Transaction.t()] + def take(amount) do + transactions_ids() + |> Enum.take(amount) + |> Enum.map(&get(&1)) + end + + @doc """ + Returns the `n` most recent transactions, unless there are not as many stored, + in which case returns `nil` + """ + @spec take_enough(integer()) :: [Transaction.t()] | nil + def take_enough(amount) do + if enough?(amount), do: take(amount) + end + + @doc """ + Adds a transaction (or a list of transactions). + If the cache is already full, the transaction will be only stored if it can take + the place of a less recent one. + NOTE: each transaction is inserted atomically + """ + @spec update([Transaction.t()] | Transaction.t() | nil) :: :ok + def update(transactions) when is_nil(transactions), do: :ok + + def update(transactions) when is_list(transactions) do + Enum.map(transactions, &update(&1)) + end + + def update(transaction) do + ConCache.isolated(@cache_name, @transactions_ids_key, fn -> + transaction_id = {transaction.block_number, transaction.index} + ids = transactions_ids() + + if full?() do + {init, [min]} = Enum.split(ids, -1) + + cond do + transaction_id < min -> + :ok + + transaction_id > min -> + insert_transaction(transaction_id, transaction, init) + ConCache.delete(@cache_name, min) + + transaction_id == min -> + put_transaction(transaction_id, transaction) + end + else + insert_transaction(transaction_id, transaction, ids) + end + end) + end + + defp insert_transaction(transaction_id, transaction, ids) do + put_transaction(transaction_id, transaction) + + ConCache.put(@cache_name, @transactions_ids_key, insert_sorted(transaction_id, ids)) + end + + defp put_transaction(transaction_id, transaction) do + full_transaction = Repo.preload(transaction, @preloads) + + ConCache.put(@cache_name, transaction_id, full_transaction) + end + + defp insert_sorted(id, ids) do + case ids do + [] -> + [id] + + [head | tail] -> + cond do + head > id -> [head | insert_sorted(id, tail)] + head < id -> [id | ids] + head == id -> ids + end + end + end +end diff --git a/apps/explorer/test/explorer/chain/import/runner/internal_transactions_test.exs b/apps/explorer/test/explorer/chain/import/runner/internal_transactions_test.exs new file mode 100644 index 0000000000..f5c080991e --- /dev/null +++ b/apps/explorer/test/explorer/chain/import/runner/internal_transactions_test.exs @@ -0,0 +1,61 @@ +defmodule Explorer.Chain.Import.Runner.InternalTransactionsTest do + use Explorer.DataCase + + alias Ecto.Multi + alias Explorer.Chain.{Data, Wei, Transaction} + alias Explorer.Chain.Import.Runner.InternalTransactions + + describe "run/1" do + test "transaction's status becomes :error when its internal_transaction has an error" do + transaction = insert(:transaction) |> with_block(status: :ok) + + assert :ok == transaction.status + + index = 0 + error = "Reverted" + + internal_transaction_changes = make_internal_transaction_changes(transaction.hash, index, error) + + assert {:ok, _} = run_internal_transactions([internal_transaction_changes]) + + assert :error == Repo.get(Transaction, transaction.hash).status + end + end + + defp run_internal_transactions(changes_list) when is_list(changes_list) do + Multi.new() + |> InternalTransactions.run(changes_list, %{ + timeout: :infinity, + timestamps: %{inserted_at: DateTime.utc_now(), updated_at: DateTime.utc_now()} + }) + |> Repo.transaction() + end + + defp make_internal_transaction_changes(transaction_hash, index, error) do + %{ + from_address_hash: insert(:address).hash, + to_address_hash: insert(:address).hash, + call_type: :call, + gas: 22234, + gas_used: + if is_nil(error) do + 18920 + else + nil + end, + input: %Data{bytes: <<1>>}, + output: + if is_nil(error) do + %Data{bytes: <<2>>} + else + nil + end, + index: index, + trace_address: [], + transaction_hash: transaction_hash, + type: :call, + value: Wei.from(Decimal.new(1), :wei), + error: error + } + end +end diff --git a/apps/explorer/test/explorer/chain/transactions_cache_test.exs b/apps/explorer/test/explorer/chain/transactions_cache_test.exs new file mode 100644 index 0000000000..a72055c276 --- /dev/null +++ b/apps/explorer/test/explorer/chain/transactions_cache_test.exs @@ -0,0 +1,95 @@ +defmodule Explorer.Chain.TransactionsCacheTest do + use Explorer.DataCase + + alias Explorer.Chain.TransactionsCache + alias Explorer.Repo + + @size 51 + + describe "update/1" do + test "adds a new value to a new cache with preloads" do + transaction = insert(:transaction) |> preload_all() + + TransactionsCache.update(transaction) + + assert TransactionsCache.take(1) == [transaction] + end + + test "adds several elements, removing the oldest when necessary" do + transactions = + 1..@size + |> Enum.map(fn n -> + block = insert(:block, number: n) + insert(:transaction) |> with_block(block) + end) + + TransactionsCache.update(transactions) + + assert TransactionsCache.all() == Enum.reverse(preload_all(transactions)) + + more_transactions = + (@size + 1)..(@size + 10) + |> Enum.map(fn n -> + block = insert(:block, number: n) + insert(:transaction) |> with_block(block) + end) + + TransactionsCache.update(more_transactions) + + kept_transactions = + Enum.reverse(transactions ++ more_transactions) + |> Enum.take(@size) + |> preload_all() + + assert TransactionsCache.take(@size) == kept_transactions + end + + test "does not add a transaction too old when full" do + transactions = + 10..(@size + 9) + |> Enum.map(fn n -> + block = insert(:block, number: n) + insert(:transaction) |> with_block(block) + end) + + TransactionsCache.update(transactions) + + loaded_transactions = Enum.reverse(preload_all(transactions)) + assert TransactionsCache.all() == loaded_transactions + + block = insert(:block, number: 1) + insert(:transaction) |> with_block(block) |> TransactionsCache.update() + + assert TransactionsCache.all() == loaded_transactions + end + + test "adds intermediate transactions" do + blocks = 1..10 |> Map.new(fn n -> {n, insert(:block, number: n)} end) + + insert(:transaction) |> with_block(blocks[1]) |> TransactionsCache.update() + insert(:transaction) |> with_block(blocks[10]) |> TransactionsCache.update() + + assert TransactionsCache.size() == 2 + + insert(:transaction) |> with_block(blocks[5]) |> TransactionsCache.update() + + assert TransactionsCache.size() == 3 + end + end + + defp preload_all(transactions) when is_list(transactions) do + Enum.map(transactions, &preload_all(&1)) + end + + defp preload_all(transaction) do + Repo.preload(transaction, [ + :block, + created_contract_address: :names, + from_address: :names, + to_address: :names, + token_transfers: :token, + token_transfers: :from_address, + token_transfers: :to_address + ]) + end +end diff --git a/apps/explorer/test/explorer/chain_test.exs b/apps/explorer/test/explorer/chain_test.exs index b2a27efbae..91a4bff8db 100644 --- a/apps/explorer/test/explorer/chain_test.exs +++ b/apps/explorer/test/explorer/chain_test.exs @@ -2110,11 +2110,14 @@ defmodule Explorer.ChainTest do ]) ) - assert internal_transaction.transaction.block.number == block.number + assert internal_transaction.transaction.block_number == block.number end test "with transaction with internal transactions loads associations with in necessity_by_association" do - transaction = insert(:transaction) + transaction = + :transaction + |> insert() + |> with_block() insert(:internal_transaction_create, transaction: transaction, @@ -2127,7 +2130,7 @@ defmodule Explorer.ChainTest do %InternalTransaction{ from_address: %Ecto.Association.NotLoaded{}, to_address: %Ecto.Association.NotLoaded{}, - transaction: %Transaction{} + transaction: %Transaction{block: %Ecto.Association.NotLoaded{}} } ] = Chain.transaction_to_internal_transactions(transaction) @@ -2135,15 +2138,15 @@ defmodule Explorer.ChainTest do %InternalTransaction{ from_address: %Address{}, to_address: nil, - transaction: %Transaction{} + transaction: %Transaction{block: %Block{}} } ] = Chain.transaction_to_internal_transactions( transaction, necessity_by_association: %{ - from_address: :optional, - to_address: :optional, - transaction: :optional + :from_address => :optional, + :to_address => :optional, + [transaction: :block] => :optional } ) end diff --git a/apps/explorer/test/support/data_case.ex b/apps/explorer/test/support/data_case.ex index c201932cef..2ec7cde365 100644 --- a/apps/explorer/test/support/data_case.ex +++ b/apps/explorer/test/support/data_case.ex @@ -42,6 +42,8 @@ defmodule Explorer.DataCase do Explorer.Chain.BlockNumberCache.setup() Supervisor.terminate_child(Explorer.Supervisor, {ConCache, Explorer.Chain.BlocksCache.cache_name()}) Supervisor.restart_child(Explorer.Supervisor, {ConCache, Explorer.Chain.BlocksCache.cache_name()}) + Supervisor.terminate_child(Explorer.Supervisor, {ConCache, Explorer.Chain.TransactionsCache.cache_name()}) + Supervisor.restart_child(Explorer.Supervisor, {ConCache, Explorer.Chain.TransactionsCache.cache_name()}) :ok end diff --git a/apps/indexer/lib/indexer/block/fetcher.ex b/apps/indexer/lib/indexer/block/fetcher.ex index 4d4cd978ec..b436d61e03 100644 --- a/apps/indexer/lib/indexer/block/fetcher.ex +++ b/apps/indexer/lib/indexer/block/fetcher.ex @@ -11,7 +11,7 @@ defmodule Indexer.Block.Fetcher do alias EthereumJSONRPC.{Blocks, FetchedBeneficiaries} alias Explorer.Chain - alias Explorer.Chain.{Address, Block, BlockNumberCache, BlocksCache, Hash, Import, Transaction} + alias Explorer.Chain.{Address, Block, BlockNumberCache, BlocksCache, Hash, Import, Transaction, TransactionsCache} alias Indexer.Block.Fetcher.Receipts alias Indexer.Fetcher.{ @@ -173,6 +173,7 @@ defmodule Indexer.Block.Fetcher do ) do result = {:ok, %{inserted: inserted, errors: blocks_errors}} update_block_cache(inserted[:blocks]) + update_transactions_cache(inserted[:transactions]) result else {step, {:error, reason}} -> {:error, {step, reason}} @@ -189,6 +190,10 @@ defmodule Indexer.Block.Fetcher do BlocksCache.update_blocks(blocks) end + defp update_transactions_cache(transactions) do + TransactionsCache.update(transactions) + end + def import( %__MODULE__{broadcast: broadcast, callback_module: callback_module} = state, options diff --git a/apps/indexer/lib/indexer/temporary/blocks_transactions_mismatch.ex b/apps/indexer/lib/indexer/temporary/blocks_transactions_mismatch.ex index 486789f675..ec1bb7f7b9 100644 --- a/apps/indexer/lib/indexer/temporary/blocks_transactions_mismatch.ex +++ b/apps/indexer/lib/indexer/temporary/blocks_transactions_mismatch.ex @@ -54,7 +54,7 @@ defmodule Indexer.Temporary.BlocksTransactionsMismatch do left_join: transactions in assoc(block, :transactions), where: block.consensus and block.refetch_needed, group_by: block.hash, - select: {block, count(transactions.hash)} + select: {block.hash, count(transactions.hash)} ) {:ok, final} = Repo.stream_reduce(query, initial, &reducer.(&1, &2)) @@ -64,7 +64,7 @@ defmodule Indexer.Temporary.BlocksTransactionsMismatch do @impl BufferedTask def run(blocks_data, json_rpc_named_arguments) do - hashes = Enum.map(blocks_data, fn {block, _trans_num} -> block.hash end) + hashes = Enum.map(blocks_data, fn {hash, _trans_num} -> hash end) Logger.debug("fetching") @@ -95,17 +95,17 @@ defmodule Indexer.Temporary.BlocksTransactionsMismatch do |> Map.merge(blocks_with_transactions_map) {found_blocks_data, missing_blocks_data} = - Enum.split_with(blocks_data, fn {block, _trans_num} -> - Map.has_key?(found_blocks_map, to_string(block.hash)) + Enum.split_with(blocks_data, fn {hash, _trans_num} -> + Map.has_key?(found_blocks_map, to_string(hash)) end) {matching_blocks_data, unmatching_blocks_data} = - Enum.split_with(found_blocks_data, fn {block, trans_num} -> - found_blocks_map[to_string(block.hash)] == trans_num + Enum.split_with(found_blocks_data, fn {hash, trans_num} -> + found_blocks_map[to_string(hash)] == trans_num end) unless Enum.empty?(matching_blocks_data) do - hashes = Enum.map(matching_blocks_data, fn {block, _trans_num} -> block.hash end) + hashes = Enum.map(matching_blocks_data, fn {hash, _trans_num} -> hash end) Block |> where([block], block.hash in ^hashes) @@ -113,7 +113,7 @@ defmodule Indexer.Temporary.BlocksTransactionsMismatch do end unless Enum.empty?(unmatching_blocks_data) do - hashes = Enum.map(unmatching_blocks_data, fn {block, _trans_num} -> block.hash end) + hashes = Enum.map(unmatching_blocks_data, fn {hash, _trans_num} -> hash end) Block |> where([block], block.hash in ^hashes)