From 44e6b6d489251984eb34a556b90b740d7c845ee6 Mon Sep 17 00:00:00 2001 From: nikitosing <32202610+nikitosing@users.noreply.github.com> Date: Mon, 5 Aug 2024 15:45:48 +0300 Subject: [PATCH] feat: Add CSV_EXPORT_LIMIT env (#10497) * feat: Add CSV_EXPORT_LIMIT env * Reuse existing function --- .../controllers/api/v2/config_controller.ex | 8 ++++++++ .../controllers/api/v2/csv_export_controller.ex | 7 ++++--- .../lib/block_scout_web/routers/api_router.ex | 1 + .../address_internal_transaction_csv_exporter.ex | 6 ++---- .../explorer/chain/csv_export/address_log_csv_exporter.ex | 6 ++---- .../csv_export/address_token_transfer_csv_exporter.ex | 6 +++--- .../chain/csv_export/address_transaction_csv_exporter.ex | 6 ++---- apps/explorer/lib/explorer/chain/csv_export/helper.ex | 7 +++++-- config/runtime.exs | 3 ++- docker-compose/envs/common-blockscout.env | 3 ++- 10 files changed, 31 insertions(+), 22 deletions(-) diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/config_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/config_controller.ex index f9d2dfc0a4..dd695a20f3 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/config_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/config_controller.ex @@ -8,4 +8,12 @@ defmodule BlockScoutWeb.API.V2.ConfigController do |> put_status(200) |> render(:backend_version, %{version: backend_version}) end + + def csv_export(conn, _params) do + limit = Application.get_env(:explorer, :csv_export_limit) + + conn + |> put_status(200) + |> json(%{limit: limit}) + end end diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/csv_export_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/csv_export_controller.ex index d8ef80f8f8..30e1fa79c2 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/csv_export_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/csv_export_controller.ex @@ -2,14 +2,13 @@ defmodule BlockScoutWeb.API.V2.CSVExportController do use BlockScoutWeb, :controller alias BlockScoutWeb.AccessHelper - alias Explorer.{Chain, PagingOptions} + alias Explorer.Chain alias Explorer.Chain.Address.CurrentTokenBalance alias Explorer.Chain.CSVExport.Helper, as: CSVHelper alias Plug.Conn action_fallback(BlockScoutWeb.API.V2.FallbackController) - @options [paging_options: %PagingOptions{page_size: CSVHelper.limit()}, api?: true] @api_true [api?: true] @doc """ @@ -25,7 +24,7 @@ defmodule BlockScoutWeb.API.V2.CSVExportController do Application.get_env(:block_scout_web, :recaptcha)[:is_disabled] || CSVHelper.captcha_helper().recaptcha_passed?(params["recaptcha_response"])}, {:not_found, {:ok, token}} <- {:not_found, Chain.token_from_address_hash(address_hash, @api_true)} do - token_holders = Chain.fetch_token_holders_from_token_hash_for_csv(address_hash, @options) + token_holders = Chain.fetch_token_holders_from_token_hash_for_csv(address_hash, options()) token_holders |> CurrentTokenBalance.to_csv_format(token) @@ -50,4 +49,6 @@ defmodule BlockScoutWeb.API.V2.CSVExportController do |> put_resp_cookie("csv-downloaded", "true", max_age: 86_400, http_only: false) |> send_chunked(200) end + + defp options, do: [paging_options: CSVHelper.paging_options(), api?: true] end diff --git a/apps/block_scout_web/lib/block_scout_web/routers/api_router.ex b/apps/block_scout_web/lib/block_scout_web/routers/api_router.ex index ff21af9fa6..3aee52a138 100644 --- a/apps/block_scout_web/lib/block_scout_web/routers/api_router.ex +++ b/apps/block_scout_web/lib/block_scout_web/routers/api_router.ex @@ -107,6 +107,7 @@ defmodule BlockScoutWeb.Routers.ApiRouter do scope "/config" do get("/backend-version", V2.ConfigController, :backend_version) + get("/csv-export", V2.ConfigController, :csv_export) end scope "/transactions" do diff --git a/apps/explorer/lib/explorer/chain/csv_export/address_internal_transaction_csv_exporter.ex b/apps/explorer/lib/explorer/chain/csv_export/address_internal_transaction_csv_exporter.ex index 4265961e98..73883fcab6 100644 --- a/apps/explorer/lib/explorer/chain/csv_export/address_internal_transaction_csv_exporter.ex +++ b/apps/explorer/lib/explorer/chain/csv_export/address_internal_transaction_csv_exporter.ex @@ -3,18 +3,16 @@ defmodule Explorer.Chain.CSVExport.AddressInternalTransactionCsvExporter do Exports internal transactions to a csv file. """ - alias Explorer.{Chain, PagingOptions} + alias Explorer.Chain alias Explorer.Chain.{Address, Hash, Transaction, Wei} alias Explorer.Chain.CSVExport.Helper - @paging_options %PagingOptions{page_size: Helper.limit()} - @spec export(Hash.Address.t(), String.t(), String.t(), String.t() | nil, String.t() | nil) :: Enumerable.t() def export(address_hash, from_period, to_period, filter_type \\ nil, filter_value \\ nil) do {from_block, to_block} = Helper.block_from_period(from_period, to_period) address_hash - |> fetch_all_internal_transactions(from_block, to_block, filter_type, filter_value, @paging_options) + |> fetch_all_internal_transactions(from_block, to_block, filter_type, filter_value, Helper.paging_options()) |> Enum.sort_by(&{&1.block_number, &1.index, &1.transaction_index}, :desc) |> to_csv_format() |> Helper.dump_to_stream() diff --git a/apps/explorer/lib/explorer/chain/csv_export/address_log_csv_exporter.ex b/apps/explorer/lib/explorer/chain/csv_export/address_log_csv_exporter.ex index 7432cb8a27..b72408b2f5 100644 --- a/apps/explorer/lib/explorer/chain/csv_export/address_log_csv_exporter.ex +++ b/apps/explorer/lib/explorer/chain/csv_export/address_log_csv_exporter.ex @@ -3,18 +3,16 @@ defmodule Explorer.Chain.CSVExport.AddressLogCsvExporter do Exports internal transactions to a csv file. """ - alias Explorer.{Chain, PagingOptions} + alias Explorer.Chain alias Explorer.Chain.{Address, Hash} alias Explorer.Chain.CSVExport.Helper - @paging_options %PagingOptions{page_size: Helper.limit()} - @spec export(Hash.Address.t(), String.t(), String.t(), String.t() | nil, String.t() | nil) :: Enumerable.t() def export(address_hash, from_period, to_period, _filter_type \\ nil, filter_value \\ nil) do {from_block, to_block} = Helper.block_from_period(from_period, to_period) address_hash - |> fetch_all_logs(from_block, to_block, filter_value, @paging_options) + |> fetch_all_logs(from_block, to_block, filter_value, Helper.paging_options()) |> to_csv_format() |> Helper.dump_to_stream() end diff --git a/apps/explorer/lib/explorer/chain/csv_export/address_token_transfer_csv_exporter.ex b/apps/explorer/lib/explorer/chain/csv_export/address_token_transfer_csv_exporter.ex index b82bd64879..8b83bcf757 100644 --- a/apps/explorer/lib/explorer/chain/csv_export/address_token_transfer_csv_exporter.ex +++ b/apps/explorer/lib/explorer/chain/csv_export/address_token_transfer_csv_exporter.ex @@ -15,14 +15,14 @@ defmodule Explorer.Chain.CSVExport.AddressTokenTransferCsvExporter do alias Explorer.Chain.{Address, DenormalizationHelper, Hash, TokenTransfer, Transaction} alias Explorer.Chain.CSVExport.Helper - @paging_options %PagingOptions{page_size: Helper.limit(), asc_order: true} - @spec export(Hash.Address.t(), String.t(), String.t(), String.t() | nil, String.t() | nil) :: Enumerable.t() def export(address_hash, from_period, to_period, filter_type \\ nil, filter_value \\ nil) do {from_block, to_block} = Helper.block_from_period(from_period, to_period) + paging_options = %PagingOptions{Helper.paging_options() | asc_order: true} + address_hash - |> fetch_all_token_transfers(from_block, to_block, filter_type, filter_value, @paging_options) + |> fetch_all_token_transfers(from_block, to_block, filter_type, filter_value, paging_options) |> to_csv_format(address_hash) |> Helper.dump_to_stream() end diff --git a/apps/explorer/lib/explorer/chain/csv_export/address_transaction_csv_exporter.ex b/apps/explorer/lib/explorer/chain/csv_export/address_transaction_csv_exporter.ex index bfe1e92cfd..d332cd7dd1 100644 --- a/apps/explorer/lib/explorer/chain/csv_export/address_transaction_csv_exporter.ex +++ b/apps/explorer/lib/explorer/chain/csv_export/address_transaction_csv_exporter.ex @@ -3,20 +3,18 @@ defmodule Explorer.Chain.CSVExport.AddressTransactionCsvExporter do Exports transactions to a csv file. """ - alias Explorer.{Market, PagingOptions} + alias Explorer.Market alias Explorer.Market.MarketHistory alias Explorer.Chain.{Address, DenormalizationHelper, Hash, Transaction, Wei} alias Explorer.Chain.CSVExport.Helper - @paging_options %PagingOptions{page_size: Helper.limit()} - @spec export(Hash.Address.t(), String.t(), String.t(), String.t() | nil, String.t() | nil) :: Enumerable.t() def export(address_hash, from_period, to_period, filter_type \\ nil, filter_value \\ nil) do {from_block, to_block} = Helper.block_from_period(from_period, to_period) exchange_rate = Market.get_coin_exchange_rate() address_hash - |> fetch_transactions(from_block, to_block, filter_type, filter_value, @paging_options) + |> fetch_transactions(from_block, to_block, filter_type, filter_value, Helper.paging_options()) |> to_csv_format(address_hash, exchange_rate) |> Helper.dump_to_stream() end diff --git a/apps/explorer/lib/explorer/chain/csv_export/helper.ex b/apps/explorer/lib/explorer/chain/csv_export/helper.ex index 53a815b79d..973f9f6097 100644 --- a/apps/explorer/lib/explorer/chain/csv_export/helper.ex +++ b/apps/explorer/lib/explorer/chain/csv_export/helper.ex @@ -12,7 +12,6 @@ defmodule Explorer.Chain.CSVExport.Helper do where: 3 ] - @limit 10_000 @page_size 150 @default_paging_options %PagingOptions{page_size: @page_size} @@ -25,7 +24,11 @@ defmodule Explorer.Chain.CSVExport.Helper do def default_paging_options, do: @default_paging_options - def limit, do: @limit + @spec limit() :: integer() + def limit, do: Application.get_env(:explorer, :csv_export_limit) + + @spec paging_options() :: Explorer.PagingOptions.t() + def paging_options, do: %PagingOptions{page_size: limit()} def block_from_period(from_period, to_period) do from_block = Chain.convert_date_to_min_block(from_period) diff --git a/config/runtime.exs b/config/runtime.exs index 10941697c3..9f4bf0005a 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -233,7 +233,8 @@ config :explorer, restricted_list_key: System.get_env("RESTRICTED_LIST_KEY"), checksum_function: checksum_function && String.to_atom(checksum_function), elasticity_multiplier: ConfigHelper.parse_integer_env_var("EIP_1559_ELASTICITY_MULTIPLIER", 2), - base_fee_max_change_denominator: ConfigHelper.parse_integer_env_var("EIP_1559_BASE_FEE_MAX_CHANGE_DENOMINATOR", 8) + base_fee_max_change_denominator: ConfigHelper.parse_integer_env_var("EIP_1559_BASE_FEE_MAX_CHANGE_DENOMINATOR", 8), + csv_export_limit: ConfigHelper.parse_integer_env_var("CSV_EXPORT_LIMIT", 10_000) config :explorer, :proxy, caching_implementation_data_enabled: true, diff --git a/docker-compose/envs/common-blockscout.env b/docker-compose/envs/common-blockscout.env index 489e32ebfe..882f37241a 100644 --- a/docker-compose/envs/common-blockscout.env +++ b/docker-compose/envs/common-blockscout.env @@ -404,4 +404,5 @@ TENDERLY_CHAIN_PATH= # SANITIZE_INCORRECT_WETH_BATCH_SIZE=100 # SANITIZE_INCORRECT_WETH_CONCURRENCY=1 # PUBLIC_METRICS_ENABLED= -# PUBLIC_METRICS_UPDATE_PERIOD_HOURS= \ No newline at end of file +# PUBLIC_METRICS_UPDATE_PERIOD_HOURS= +# CSV_EXPORT_LIMIT= \ No newline at end of file