feat: Add CSV_EXPORT_LIMIT env (#10497)

* feat: Add CSV_EXPORT_LIMIT env

* Reuse existing function

apps/block_scout_web/lib/block_scout_web/controllers/api/v2/config_controller.ex

@@ -8,4 +8,12 @@ defmodule BlockScoutWeb.API.V2.ConfigController do
     |> put_status(200)
     |> render(:backend_version, %{version: backend_version})
   end
+
+  def csv_export(conn, _params) do
+    limit = Application.get_env(:explorer, :csv_export_limit)
+
+    conn
+    |> put_status(200)
+    |> json(%{limit: limit})
+  end
 end

apps/block_scout_web/lib/block_scout_web/controllers/api/v2/csv_export_controller.ex

@@ -2,14 +2,13 @@ defmodule BlockScoutWeb.API.V2.CSVExportController do
   use BlockScoutWeb, :controller
 
   alias BlockScoutWeb.AccessHelper
-  alias Explorer.{Chain, PagingOptions}
+  alias Explorer.Chain
   alias Explorer.Chain.Address.CurrentTokenBalance
   alias Explorer.Chain.CSVExport.Helper, as: CSVHelper
   alias Plug.Conn
 
   action_fallback(BlockScoutWeb.API.V2.FallbackController)
 
-  @options [paging_options: %PagingOptions{page_size: CSVHelper.limit()}, api?: true]
   @api_true [api?: true]
 
   @doc """
@@ -25,7 +24,7 @@ defmodule BlockScoutWeb.API.V2.CSVExportController do
             Application.get_env(:block_scout_web, :recaptcha)[:is_disabled] ||
               CSVHelper.captcha_helper().recaptcha_passed?(params["recaptcha_response"])},
          {:not_found, {:ok, token}} <- {:not_found, Chain.token_from_address_hash(address_hash, @api_true)} do
-      token_holders = Chain.fetch_token_holders_from_token_hash_for_csv(address_hash, @options)
+      token_holders = Chain.fetch_token_holders_from_token_hash_for_csv(address_hash, options())
 
       token_holders
       |> CurrentTokenBalance.to_csv_format(token)
@@ -50,4 +49,6 @@ defmodule BlockScoutWeb.API.V2.CSVExportController do
     |> put_resp_cookie("csv-downloaded", "true", max_age: 86_400, http_only: false)
     |> send_chunked(200)
   end
+
+  defp options, do: [paging_options: CSVHelper.paging_options(), api?: true]
 end

apps/block_scout_web/lib/block_scout_web/routers/api_router.ex

@@ -107,6 +107,7 @@ defmodule BlockScoutWeb.Routers.ApiRouter do
 
     scope "/config" do
       get("/backend-version", V2.ConfigController, :backend_version)
+      get("/csv-export", V2.ConfigController, :csv_export)
     end
 
     scope "/transactions" do

apps/explorer/lib/explorer/chain/csv_export/address_internal_transaction_csv_exporter.ex

@@ -3,18 +3,16 @@ defmodule Explorer.Chain.CSVExport.AddressInternalTransactionCsvExporter do
   Exports internal transactions to a csv file.
   """
 
-  alias Explorer.{Chain, PagingOptions}
+  alias Explorer.Chain
   alias Explorer.Chain.{Address, Hash, Transaction, Wei}
   alias Explorer.Chain.CSVExport.Helper
 
-  @paging_options %PagingOptions{page_size: Helper.limit()}
-
   @spec export(Hash.Address.t(), String.t(), String.t(), String.t() | nil, String.t() | nil) :: Enumerable.t()
   def export(address_hash, from_period, to_period, filter_type \\ nil, filter_value \\ nil) do
     {from_block, to_block} = Helper.block_from_period(from_period, to_period)
 
     address_hash
-    |> fetch_all_internal_transactions(from_block, to_block, filter_type, filter_value, @paging_options)
+    |> fetch_all_internal_transactions(from_block, to_block, filter_type, filter_value, Helper.paging_options())
     |> Enum.sort_by(&{&1.block_number, &1.index, &1.transaction_index}, :desc)
     |> to_csv_format()
     |> Helper.dump_to_stream()

apps/explorer/lib/explorer/chain/csv_export/address_log_csv_exporter.ex

@@ -3,18 +3,16 @@ defmodule Explorer.Chain.CSVExport.AddressLogCsvExporter do
   Exports internal transactions to a csv file.
   """
 
-  alias Explorer.{Chain, PagingOptions}
+  alias Explorer.Chain
   alias Explorer.Chain.{Address, Hash}
   alias Explorer.Chain.CSVExport.Helper
 
-  @paging_options %PagingOptions{page_size: Helper.limit()}
-
   @spec export(Hash.Address.t(), String.t(), String.t(), String.t() | nil, String.t() | nil) :: Enumerable.t()
   def export(address_hash, from_period, to_period, _filter_type \\ nil, filter_value \\ nil) do
     {from_block, to_block} = Helper.block_from_period(from_period, to_period)
 
     address_hash
-    |> fetch_all_logs(from_block, to_block, filter_value, @paging_options)
+    |> fetch_all_logs(from_block, to_block, filter_value, Helper.paging_options())
     |> to_csv_format()
     |> Helper.dump_to_stream()
   end

apps/explorer/lib/explorer/chain/csv_export/address_token_transfer_csv_exporter.ex

@@ -15,14 +15,14 @@ defmodule Explorer.Chain.CSVExport.AddressTokenTransferCsvExporter do
   alias Explorer.Chain.{Address, DenormalizationHelper, Hash, TokenTransfer, Transaction}
   alias Explorer.Chain.CSVExport.Helper
 
-  @paging_options %PagingOptions{page_size: Helper.limit(), asc_order: true}
-
   @spec export(Hash.Address.t(), String.t(), String.t(), String.t() | nil, String.t() | nil) :: Enumerable.t()
   def export(address_hash, from_period, to_period, filter_type \\ nil, filter_value \\ nil) do
     {from_block, to_block} = Helper.block_from_period(from_period, to_period)
 
+    paging_options = %PagingOptions{Helper.paging_options() | asc_order: true}
+
     address_hash
-    |> fetch_all_token_transfers(from_block, to_block, filter_type, filter_value, @paging_options)
+    |> fetch_all_token_transfers(from_block, to_block, filter_type, filter_value, paging_options)
     |> to_csv_format(address_hash)
     |> Helper.dump_to_stream()
   end

apps/explorer/lib/explorer/chain/csv_export/address_transaction_csv_exporter.ex

@@ -3,20 +3,18 @@ defmodule Explorer.Chain.CSVExport.AddressTransactionCsvExporter do
   Exports transactions to a csv file.
   """
 
-  alias Explorer.{Market, PagingOptions}
+  alias Explorer.Market
   alias Explorer.Market.MarketHistory
   alias Explorer.Chain.{Address, DenormalizationHelper, Hash, Transaction, Wei}
   alias Explorer.Chain.CSVExport.Helper
 
-  @paging_options %PagingOptions{page_size: Helper.limit()}
-
   @spec export(Hash.Address.t(), String.t(), String.t(), String.t() | nil, String.t() | nil) :: Enumerable.t()
   def export(address_hash, from_period, to_period, filter_type \\ nil, filter_value \\ nil) do
     {from_block, to_block} = Helper.block_from_period(from_period, to_period)
     exchange_rate = Market.get_coin_exchange_rate()
 
     address_hash
-    |> fetch_transactions(from_block, to_block, filter_type, filter_value, @paging_options)
+    |> fetch_transactions(from_block, to_block, filter_type, filter_value, Helper.paging_options())
     |> to_csv_format(address_hash, exchange_rate)
     |> Helper.dump_to_stream()
   end

apps/explorer/lib/explorer/chain/csv_export/helper.ex

@@ -12,7 +12,6 @@ defmodule Explorer.Chain.CSVExport.Helper do
       where: 3
     ]
 
-  @limit 10_000
   @page_size 150
   @default_paging_options %PagingOptions{page_size: @page_size}
 
@@ -25,7 +24,11 @@ defmodule Explorer.Chain.CSVExport.Helper do
 
   def default_paging_options, do: @default_paging_options
 
-  def limit, do: @limit
+  @spec limit() :: integer()
+  def limit, do: Application.get_env(:explorer, :csv_export_limit)
+
+  @spec paging_options() :: Explorer.PagingOptions.t()
+  def paging_options, do: %PagingOptions{page_size: limit()}
 
   def block_from_period(from_period, to_period) do
     from_block = Chain.convert_date_to_min_block(from_period)

config/runtime.exs

@@ -233,7 +233,8 @@ config :explorer,
   restricted_list_key: System.get_env("RESTRICTED_LIST_KEY"),
   checksum_function: checksum_function && String.to_atom(checksum_function),
   elasticity_multiplier: ConfigHelper.parse_integer_env_var("EIP_1559_ELASTICITY_MULTIPLIER", 2),
-  base_fee_max_change_denominator: ConfigHelper.parse_integer_env_var("EIP_1559_BASE_FEE_MAX_CHANGE_DENOMINATOR", 8)
+  base_fee_max_change_denominator: ConfigHelper.parse_integer_env_var("EIP_1559_BASE_FEE_MAX_CHANGE_DENOMINATOR", 8),
+  csv_export_limit: ConfigHelper.parse_integer_env_var("CSV_EXPORT_LIMIT", 10_000)
 
 config :explorer, :proxy,
   caching_implementation_data_enabled: true,

docker-compose/envs/common-blockscout.env

@@ -404,4 +404,5 @@ TENDERLY_CHAIN_PATH=
 # SANITIZE_INCORRECT_WETH_BATCH_SIZE=100
 # SANITIZE_INCORRECT_WETH_CONCURRENCY=1
 # PUBLIC_METRICS_ENABLED=
-# PUBLIC_METRICS_UPDATE_PERIOD_HOURS=
+# PUBLIC_METRICS_UPDATE_PERIOD_HOURS=
+# CSV_EXPORT_LIMIT=