From dea361d56fac753d0695eee52be7563add104ce1 Mon Sep 17 00:00:00 2001 From: Maxim Filonov <53992153+sl1depengwyn@users.noreply.github.com> Date: Fri, 15 Mar 2024 17:32:50 +0300 Subject: [PATCH] Add secondary coin and transaction stats (#9483) * Add volume_24h * Add secondary coin and transactions stats * Process review comments * Allow different source for secondary coin * Fix exchange_rates_secondary_coin_price_source --------- Co-authored-by: Nikita Pozdniakov --- CHANGELOG.md | 1 + .../lib/block_scout_web/api_router.ex | 2 + .../controllers/api/v2/stats_controller.ex | 12 ++ .../api/v2/transaction_controller.ex | 20 +++ .../views/api/v2/token_view.ex | 1 + .../views/api/v2/transaction_view.ex | 14 ++ apps/explorer/config/runtime/test.exs | 2 + apps/explorer/lib/explorer/application.ex | 2 + apps/explorer/lib/explorer/chain/token.ex | 3 +- .../fresh_pending_transactions_counter.ex | 95 ++++++++++++ .../counters/transactions_24h_stats.ex | 143 ++++++++++++++++++ .../exchange_rates/source/coin_gecko.ex | 20 ++- .../exchange_rates/source/coin_market_cap.ex | 6 + .../lib/explorer/market/history/cataloger.ex | 65 ++++++-- .../explorer/market/history/source/price.ex | 6 +- .../market/history/source/price/coin_gecko.ex | 15 +- .../history/source/price/coin_market_cap.ex | 18 ++- .../history/source/price/crypto_compare.ex | 26 ++-- apps/explorer/lib/explorer/market/market.ex | 8 +- .../lib/explorer/market/market_history.ex | 1 + .../explorer/market/market_history_cache.ex | 7 +- ...0240219143204_add_volume_24h_to_tokens.exs | 9 ++ ...1331_add_secondary_coin_market_history.exs | 12 ++ ...resh_pending_transactions_counter_test.exs | 27 ++++ .../counters/transactions_24h_stats_test.exs | 61 ++++++++ .../exchange_rates/source/coin_gecko_test.exs | 2 +- .../token_exchange_rates_test.exs | 12 +- .../market/history/cataloger_test.exs | 100 ++++++++++-- .../source/price/crypto_compare_test.exs | 22 ++- .../test/support/fakes/no_op_price_source.ex | 2 +- config/config_helper.exs | 14 ++ config/runtime.exs | 32 +++- config/runtime/test.exs | 4 + cspell.json | 7 + docker-compose/envs/common-blockscout.env | 5 + 35 files changed, 692 insertions(+), 84 deletions(-) create mode 100644 apps/explorer/lib/explorer/counters/fresh_pending_transactions_counter.ex create mode 100644 apps/explorer/lib/explorer/counters/transactions_24h_stats.ex create mode 100644 apps/explorer/priv/repo/migrations/20240219143204_add_volume_24h_to_tokens.exs create mode 100644 apps/explorer/priv/repo/migrations/20240226151331_add_secondary_coin_market_history.exs create mode 100644 apps/explorer/test/explorer/counters/fresh_pending_transactions_counter_test.exs create mode 100644 apps/explorer/test/explorer/counters/transactions_24h_stats_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index 9bff8b0f37..1d6710ed8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - [#9511](https://github.com/blockscout/blockscout/pull/9511) - Separate errors by type in EndpointAvailabilityObserver - [#9490](https://github.com/blockscout/blockscout/pull/9490), [#9644](https://github.com/blockscout/blockscout/pull/9644) - Add blob transaction counter and filter in block view - [#9486](https://github.com/blockscout/blockscout/pull/9486) - Massive blocks fetcher +- [#9483](https://github.com/blockscout/blockscout/pull/9483) - Add secondary coin and transaction stats - [#9473](https://github.com/blockscout/blockscout/pull/9473) - Add user_op interpretation - [#9461](https://github.com/blockscout/blockscout/pull/9461) - Fetch blocks without internal transactions backwards - [#9460](https://github.com/blockscout/blockscout/pull/9460) - Optimism chain type diff --git a/apps/block_scout_web/lib/block_scout_web/api_router.ex b/apps/block_scout_web/lib/block_scout_web/api_router.ex index b377a60fd9..f733afcc59 100644 --- a/apps/block_scout_web/lib/block_scout_web/api_router.ex +++ b/apps/block_scout_web/lib/block_scout_web/api_router.ex @@ -202,6 +202,7 @@ defmodule BlockScoutWeb.ApiRouter do scope "/transactions" do get("/", V2.TransactionController, :transactions) get("/watchlist", V2.TransactionController, :watchlist_transactions) + get("/stats", V2.TransactionController, :stats) if Application.compile_env(:explorer, :chain_type) == "polygon_zkevm" do get("/zkevm-batch/:batch_number", V2.TransactionController, :polygon_zkevm_batch) @@ -298,6 +299,7 @@ defmodule BlockScoutWeb.ApiRouter do scope "/charts" do get("/transactions", V2.StatsController, :transactions_chart) get("/market", V2.StatsController, :market_chart) + get("/secondary-coin-market", V2.StatsController, :secondary_coin_market_chart) end end diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/stats_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/stats_controller.ex index f4bb234f24..8cf9adbe5d 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/stats_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/stats_controller.ex @@ -147,6 +147,18 @@ defmodule BlockScoutWeb.API.V2.StatsController do }) end + def secondary_coin_market_chart(conn, _params) do + recent_market_history = Market.fetch_recent_history(true) + + chart_data = + recent_market_history + |> Enum.map(fn day -> Map.take(day, [:closing_price, :date]) end) + + json(conn, %{ + chart_data: chart_data + }) + end + defp backward_compatibility(response, conn) do case Conn.get_req_header(conn, "updated-gas-oracle") do ["true"] -> diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/transaction_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/transaction_controller.ex index be5c58bedd..05039db280 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/transaction_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/transaction_controller.ex @@ -33,6 +33,7 @@ defmodule BlockScoutWeb.API.V2.TransactionController do alias Explorer.Chain.{Hash, Transaction} alias Explorer.Chain.PolygonZkevm.Reader alias Explorer.Chain.ZkSync.Reader + alias Explorer.Counters.{FreshPendingTransactionsCounter, Transactions24hStats} alias Indexer.Fetcher.FirstTraceOnDemand action_fallback(BlockScoutWeb.API.V2.FallbackController) @@ -446,6 +447,25 @@ defmodule BlockScoutWeb.API.V2.TransactionController do end end + def stats(conn, _params) do + transactions_count = Transactions24hStats.fetch_count(@api_true) + pending_transactions_count = FreshPendingTransactionsCounter.fetch(@api_true) + transaction_fees_sum = Transactions24hStats.fetch_fee_sum(@api_true) + transaction_fees_avg = Transactions24hStats.fetch_fee_average(@api_true) + + conn + |> put_status(200) + |> render( + :stats, + %{ + transactions_count_24h: transactions_count, + pending_transactions_count: pending_transactions_count, + transaction_fees_sum_24h: transaction_fees_sum, + transaction_fees_avg_24h: transaction_fees_avg + } + ) + end + @doc """ Checks if this valid transaction hash string, and this transaction doesn't belong to prohibited address """ diff --git a/apps/block_scout_web/lib/block_scout_web/views/api/v2/token_view.ex b/apps/block_scout_web/lib/block_scout_web/views/api/v2/token_view.ex index a6b5fc99f7..f1ed2d9953 100644 --- a/apps/block_scout_web/lib/block_scout_web/views/api/v2/token_view.ex +++ b/apps/block_scout_web/lib/block_scout_web/views/api/v2/token_view.ex @@ -36,6 +36,7 @@ defmodule BlockScoutWeb.API.V2.TokenView do "type" => token.type, "holders" => prepare_holders_count(token.holder_count), "exchange_rate" => exchange_rate(token), + "volume_24h" => token.volume_24h, "total_supply" => token.total_supply, "icon_url" => token.icon_url, "circulating_market_cap" => token.circulating_market_cap diff --git a/apps/block_scout_web/lib/block_scout_web/views/api/v2/transaction_view.ex b/apps/block_scout_web/lib/block_scout_web/views/api/v2/transaction_view.ex index cd8d57e3c3..2c4c2fc478 100644 --- a/apps/block_scout_web/lib/block_scout_web/views/api/v2/transaction_view.ex +++ b/apps/block_scout_web/lib/block_scout_web/views/api/v2/transaction_view.ex @@ -184,6 +184,20 @@ defmodule BlockScoutWeb.API.V2.TransactionView do } end + def render("stats.json", %{ + transactions_count_24h: transactions_count, + pending_transactions_count: pending_transactions_count, + transaction_fees_sum_24h: transaction_fees_sum, + transaction_fees_avg_24h: transaction_fees_avg + }) do + %{ + "transactions_count_24h" => transactions_count, + "pending_transactions_count" => pending_transactions_count, + "transaction_fees_sum_24h" => transaction_fees_sum, + "transaction_fees_avg_24h" => transaction_fees_avg + } + end + @doc """ Decodes list of logs """ diff --git a/apps/explorer/config/runtime/test.exs b/apps/explorer/config/runtime/test.exs index c54b712937..d9cfc1a821 100644 --- a/apps/explorer/config/runtime/test.exs +++ b/apps/explorer/config/runtime/test.exs @@ -19,6 +19,8 @@ config :explorer, Explorer.Market.History.Historian, enabled: false config :explorer, Explorer.Counters.AddressesCounter, enabled: false, enable_consolidation: false config :explorer, Explorer.Counters.LastOutputRootSizeCounter, enabled: false, enable_consolidation: false +config :explorer, Explorer.Counters.Transactions24hStats, enabled: false, enable_consolidation: false +config :explorer, Explorer.Counters.FreshPendingTransactionsCounter, enabled: false, enable_consolidation: false config :explorer, Explorer.Chain.Cache.ContractsCounter, enabled: false, enable_consolidation: false config :explorer, Explorer.Chain.Cache.NewContractsCounter, enabled: false, enable_consolidation: false config :explorer, Explorer.Chain.Cache.VerifiedContractsCounter, enabled: false, enable_consolidation: false diff --git a/apps/explorer/lib/explorer/application.ex b/apps/explorer/lib/explorer/application.ex index b5d33647d2..96876b008b 100644 --- a/apps/explorer/lib/explorer/application.ex +++ b/apps/explorer/lib/explorer/application.ex @@ -119,6 +119,8 @@ defmodule Explorer.Application do configure(Explorer.Counters.BlockPriorityFeeCounter), configure(Explorer.Counters.AverageBlockTime), configure(Explorer.Counters.LastOutputRootSizeCounter), + configure(Explorer.Counters.FreshPendingTransactionsCounter), + configure(Explorer.Counters.Transactions24hStats), configure(Explorer.Validator.MetadataProcessor), configure(Explorer.Tags.AddressTag.Cataloger), configure(MinMissingBlockNumber), diff --git a/apps/explorer/lib/explorer/chain/token.ex b/apps/explorer/lib/explorer/chain/token.ex index 9800f462f0..6578a1bdca 100644 --- a/apps/explorer/lib/explorer/chain/token.ex +++ b/apps/explorer/lib/explorer/chain/token.ex @@ -30,6 +30,7 @@ defmodule Explorer.Chain.Token.Schema do field(:circulating_market_cap, :decimal) field(:icon_url, :string) field(:is_verified_via_admin_panel, :boolean) + field(:volume_24h, :decimal) belongs_to( :contract_address, @@ -123,7 +124,7 @@ defmodule Explorer.Chain.Token do Explorer.Chain.Token.Schema.generate() @required_attrs ~w(contract_address_hash type)a - @optional_attrs ~w(cataloged decimals name symbol total_supply skip_metadata total_supply_updated_at_block updated_at fiat_value circulating_market_cap icon_url is_verified_via_admin_panel)a + @optional_attrs ~w(cataloged decimals name symbol total_supply skip_metadata total_supply_updated_at_block updated_at fiat_value circulating_market_cap icon_url is_verified_via_admin_panel volume_24h)a @doc false def changeset(%Token{} = token, params \\ %{}) do diff --git a/apps/explorer/lib/explorer/counters/fresh_pending_transactions_counter.ex b/apps/explorer/lib/explorer/counters/fresh_pending_transactions_counter.ex new file mode 100644 index 0000000000..3a4b1548ff --- /dev/null +++ b/apps/explorer/lib/explorer/counters/fresh_pending_transactions_counter.ex @@ -0,0 +1,95 @@ +defmodule Explorer.Counters.FreshPendingTransactionsCounter do + @moduledoc """ + Caches number of pending transactions for last 30 minutes. + + It loads the sum asynchronously and in a time interval of :cache_period (default to 5 minutes). + """ + + use GenServer + + import Ecto.Query + + alias Explorer.{Chain, Repo} + alias Explorer.Chain.Transaction + + @counter_type "pending_transaction_count_30min" + + @doc """ + Starts a process to periodically update the counter. + """ + @spec start_link(term()) :: GenServer.on_start() + def start_link(_) do + GenServer.start_link(__MODULE__, :ok, name: __MODULE__) + end + + @impl true + def init(_args) do + {:ok, %{consolidate?: enable_consolidation?()}, {:continue, :ok}} + end + + defp schedule_next_consolidation do + Process.send_after(self(), :consolidate, cache_interval()) + end + + @impl true + def handle_continue(:ok, %{consolidate?: true} = state) do + consolidate() + schedule_next_consolidation() + + {:noreply, state} + end + + @impl true + def handle_continue(:ok, state) do + {:noreply, state} + end + + @impl true + def handle_info(:consolidate, state) do + consolidate() + schedule_next_consolidation() + + {:noreply, state} + end + + @doc """ + Fetches the value for a `#{@counter_type}` counter type from the `last_fetched_counters` table. + """ + def fetch(options) do + Chain.get_last_fetched_counter(@counter_type, options) + end + + @doc """ + Consolidates the info by populating the `last_fetched_counters` table with the current database information. + """ + def consolidate do + query = + from(transaction in Transaction, + where: is_nil(transaction.block_hash) and transaction.inserted_at >= ago(30, "minute"), + select: count(transaction.hash) + ) + + count = Repo.one!(query, timeout: :infinity) + + Chain.upsert_last_fetched_counter(%{ + counter_type: @counter_type, + value: count + }) + end + + @doc """ + Returns a boolean that indicates whether consolidation is enabled + + In order to choose whether or not to enable the scheduler and the initial + consolidation, change the following Explorer config: + + `config :explorer, #{__MODULE__}, enable_consolidation: true` + + to: + + `config :explorer, #{__MODULE__}, enable_consolidation: false` + """ + def enable_consolidation?, do: Application.get_env(:explorer, __MODULE__)[:enable_consolidation] + + defp cache_interval, do: Application.get_env(:explorer, __MODULE__)[:cache_period] +end diff --git a/apps/explorer/lib/explorer/counters/transactions_24h_stats.ex b/apps/explorer/lib/explorer/counters/transactions_24h_stats.ex new file mode 100644 index 0000000000..80bef49ec0 --- /dev/null +++ b/apps/explorer/lib/explorer/counters/transactions_24h_stats.ex @@ -0,0 +1,143 @@ +defmodule Explorer.Counters.Transactions24hStats do + @moduledoc """ + Caches number of transactions for last 24 hours, sum of transaction fees for last 24 hours and average transaction fee for last 24 hours counters. + + It loads the counters asynchronously and in a time interval of :cache_period (default to 1 hour). + """ + + use GenServer + + import Ecto.Query + + alias Explorer.{Chain, Repo} + alias Explorer.Chain.Transaction + + @tx_count_name "transaction_count_24h" + @tx_fee_sum_name "transaction_fee_sum_24h" + @tx_fee_average_name "transaction_fee_average_24h" + + @doc """ + Starts a process to periodically update the counters. + """ + @spec start_link(term()) :: GenServer.on_start() + def start_link(_) do + GenServer.start_link(__MODULE__, :ok, name: __MODULE__) + end + + @impl true + def init(_args) do + {:ok, %{consolidate?: enable_consolidation?()}, {:continue, :ok}} + end + + defp schedule_next_consolidation do + Process.send_after(self(), :consolidate, cache_interval()) + end + + @impl true + def handle_continue(:ok, %{consolidate?: true} = state) do + consolidate() + schedule_next_consolidation() + + {:noreply, state} + end + + @impl true + def handle_continue(:ok, state) do + {:noreply, state} + end + + @impl true + def handle_info(:consolidate, state) do + consolidate() + schedule_next_consolidation() + + {:noreply, state} + end + + @doc """ + Fetches the value for a `#{@tx_count_name}` counter type from the `last_fetched_counters` table. + """ + def fetch_count(options) do + Chain.get_last_fetched_counter(@tx_count_name, options) + end + + @doc """ + Fetches the value for a `#{@tx_fee_sum_name}` counter type from the `last_fetched_counters` table. + """ + def fetch_fee_sum(options) do + Chain.get_last_fetched_counter(@tx_fee_sum_name, options) + end + + @doc """ + Fetches the value for a `#{@tx_fee_average_name}` counter type from the `last_fetched_counters` table. + """ + def fetch_fee_average(options) do + Chain.get_last_fetched_counter(@tx_fee_average_name, options) + end + + @doc """ + Consolidates the info by populating the `last_fetched_counters` table with the current database information. + """ + def consolidate do + fee_query = + dynamic( + [transaction, block], + fragment( + "COALESCE(?, ? + LEAST(?, ?))", + transaction.gas_price, + block.base_fee_per_gas, + transaction.max_priority_fee_per_gas, + transaction.max_fee_per_gas - block.base_fee_per_gas + ) * transaction.gas_used + ) + + sum_query = dynamic([_, _], sum(^fee_query)) + avg_query = dynamic([_, _], avg(^fee_query)) + + query = + from(transaction in Transaction, + join: block in assoc(transaction, :block), + where: block.timestamp >= ago(24, "hour"), + select: %{count: count(transaction.hash)}, + select_merge: ^%{fee_sum: sum_query}, + select_merge: ^%{fee_average: avg_query} + ) + + %{ + count: count, + fee_sum: fee_sum, + fee_average: fee_average + } = Repo.one!(query, timeout: :infinity) + + Chain.upsert_last_fetched_counter(%{ + counter_type: @tx_count_name, + value: count + }) + + Chain.upsert_last_fetched_counter(%{ + counter_type: @tx_fee_sum_name, + value: fee_sum + }) + + Chain.upsert_last_fetched_counter(%{ + counter_type: @tx_fee_average_name, + value: fee_average + }) + end + + @doc """ + Returns a boolean that indicates whether consolidation is enabled + + In order to choose whether or not to enable the scheduler and the initial + consolidation, change the following Explorer config: + + `config :explorer, #{__MODULE__}, enable_consolidation: true` + + to: + + `config :explorer, #{__MODULE__}, enable_consolidation: false` + """ + def enable_consolidation?, do: Application.get_env(:explorer, __MODULE__)[:enable_consolidation] + + defp cache_interval, do: Application.get_env(:explorer, __MODULE__)[:cache_period] +end diff --git a/apps/explorer/lib/explorer/exchange_rates/source/coin_gecko.ex b/apps/explorer/lib/explorer/exchange_rates/source/coin_gecko.ex index a1282f525d..40076e6f63 100644 --- a/apps/explorer/lib/explorer/exchange_rates/source/coin_gecko.ex +++ b/apps/explorer/lib/explorer/exchange_rates/source/coin_gecko.ex @@ -51,6 +51,7 @@ defmodule Explorer.ExchangeRates.Source.CoinGecko do def format_data(%{} = market_data_for_tokens) do currency = currency() market_cap = currency <> "_market_cap" + volume_24h = currency <> "_24h_vol" market_data_for_tokens |> Enum.reduce(%{}, fn @@ -60,7 +61,8 @@ defmodule Explorer.ExchangeRates.Source.CoinGecko do acc |> Map.put(address_hash, %{ fiat_value: Map.get(market_data, currency), - circulating_market_cap: Map.get(market_data, market_cap) + circulating_market_cap: Map.get(market_data, market_cap), + volume_24h: Map.get(market_data, volume_24h) }) _ -> @@ -92,14 +94,22 @@ defmodule Explorer.ExchangeRates.Source.CoinGecko do @impl Source def format_data(_), do: [] - @spec history_url(non_neg_integer()) :: String.t() - def history_url(previous_days) do + @spec history_url(non_neg_integer(), boolean()) :: String.t() + def history_url(previous_days, secondary_coin? \\ false) do query_params = %{ "days" => previous_days, "vs_currency" => "usd" } - "#{source_url()}/market_chart?#{URI.encode_query(query_params)}" + source_url = if secondary_coin?, do: secondary_source_url(), else: source_url() + + "#{source_url}/market_chart?#{URI.encode_query(query_params)}" + end + + def secondary_source_url do + id = config(:secondary_coin_id) + + if id, do: "#{base_url()}/coins/#{id}", else: nil end @impl Source @@ -131,7 +141,7 @@ defmodule Explorer.ExchangeRates.Source.CoinGecko do def source_url(token_addresses) when is_list(token_addresses) do joined_addresses = token_addresses |> Enum.map_join(",", &to_string/1) - "#{base_url()}/simple/token_price/#{platform()}?vs_currencies=#{currency()}&include_market_cap=true&contract_addresses=#{joined_addresses}" + "#{base_url()}/simple/token_price/#{platform()}?vs_currencies=#{currency()}&include_market_cap=true&include_24hr_vol=true&contract_addresses=#{joined_addresses}" end @impl Source diff --git a/apps/explorer/lib/explorer/exchange_rates/source/coin_market_cap.ex b/apps/explorer/lib/explorer/exchange_rates/source/coin_market_cap.ex index 2e153e4dee..739d1c0425 100644 --- a/apps/explorer/lib/explorer/exchange_rates/source/coin_market_cap.ex +++ b/apps/explorer/lib/explorer/exchange_rates/source/coin_market_cap.ex @@ -71,6 +71,12 @@ defmodule Explorer.ExchangeRates.Source.CoinMarketCap do end end + @impl Source + def source_url(:secondary_coin) do + coin_id = config(:secondary_coin_id) + if coin_id, do: "#{api_quotes_latest_url()}?id=#{coin_id}&CMC_PRO_API_KEY=#{api_key()}", else: nil + end + @impl Source def source_url(input) do case Chain.Hash.Address.cast(input) do diff --git a/apps/explorer/lib/explorer/market/history/cataloger.ex b/apps/explorer/lib/explorer/market/history/cataloger.ex index e52052a711..19b0c7bc4f 100644 --- a/apps/explorer/lib/explorer/market/history/cataloger.ex +++ b/apps/explorer/lib/explorer/market/history/cataloger.ex @@ -40,13 +40,33 @@ defmodule Explorer.Market.History.Cataloger do @impl GenServer # Record fetch successful. - def handle_info({_ref, {:price_history, {_, _, {:ok, records}}}}, state) do - Process.send(self(), {:fetch_market_cap_history, 365}, []) + def handle_info({_ref, {:price_history, {day_count, _, false, {:ok, records}}}}, state) do + if config_or_default(:secondary_coin_enabled, false) do + Process.send(self(), {:fetch_price_history_for_secondary_coin, day_count}, []) + else + Process.send(self(), {:fetch_market_cap_history, day_count}, []) + end + state = state |> Map.put_new(:price_records, records) {:noreply, state} end + # Secondary coin. + def handle_info({_ref, {:price_history, {day_count, _, true, {:ok, records}}}}, state) do + Process.send(self(), {:fetch_market_cap_history, day_count}, []) + state = state |> Map.put_new(:secondary_coin_price_records, records) + + {:noreply, state} + end + + @impl GenServer + def handle_info({:fetch_price_history_for_secondary_coin, day_count}, state) do + fetch_price_history(day_count, true) + + {:noreply, state} + end + @impl GenServer def handle_info({:fetch_market_cap_history, day_count}, state) do fetch_market_cap_history(day_count) @@ -98,10 +118,10 @@ defmodule Explorer.Market.History.Cataloger do # Failed to get records. Try again. @impl GenServer - def handle_info({_ref, {:price_history, {day_count, failed_attempts, :error}}}, state) do + def handle_info({_ref, {:price_history, {day_count, failed_attempts, secondary_coin?, :error}}}, state) do Logger.warn(fn -> "Failed to fetch price history. Trying again." end) - fetch_price_history(day_count, failed_attempts + 1) + fetch_price_history(day_count, secondary_coin?, failed_attempts + 1) {:noreply, state} end @@ -150,19 +170,31 @@ defmodule Explorer.Market.History.Cataloger do Application.get_env(:explorer, __MODULE__)[key] || default end - defp market_cap_history(records, state) do + defp market_cap_history(records, _state) do Market.bulk_insert_history(records) # Schedule next check for history fetch_after = config_or_default(:history_fetch_interval, :timer.minutes(60)) Process.send_after(self(), {:fetch_price_history, 1}, fetch_after) - {:noreply, state} + {:noreply, %{}} end - @spec source_price() :: module() - defp source_price do - config_or_default(:price_source, Explorer.ExchangeRates.Source, Explorer.Market.History.Source.Price.CryptoCompare) + @spec source_price(boolean()) :: module() + defp source_price(secondary_coin?) do + if secondary_coin? do + config_or_default( + :secondary_coin_price_source, + Explorer.ExchangeRates.Source, + Explorer.Market.History.Source.Price.CryptoCompare + ) + else + config_or_default( + :price_source, + Explorer.ExchangeRates.Source, + Explorer.Market.History.Source.Price.CryptoCompare + ) + end end @spec source_market_cap() :: module() @@ -183,15 +215,17 @@ defmodule Explorer.Market.History.Cataloger do ) end - @spec fetch_price_history(non_neg_integer(), non_neg_integer()) :: Task.t() - defp fetch_price_history(day_count, failed_attempts \\ 0) do + @spec fetch_price_history(non_neg_integer(), boolean(), non_neg_integer()) :: Task.t() + defp fetch_price_history(day_count, secondary_coin? \\ false, failed_attempts \\ 0) do Task.Supervisor.async_nolink(Explorer.MarketTaskSupervisor, fn -> Process.sleep(HistoryProcess.delay(failed_attempts)) if failed_attempts < @price_failed_attempts do - {:price_history, {day_count, failed_attempts, source_price().fetch_price_history(day_count)}} + {:price_history, + {day_count, failed_attempts, secondary_coin?, + source_price(secondary_coin?).fetch_price_history(day_count, secondary_coin?)}} else - {:price_history, {day_count, failed_attempts, {:ok, []}}} + {:price_history, {day_count, failed_attempts, secondary_coin?, {:ok, []}}} end end) end @@ -224,13 +258,14 @@ defmodule Explorer.Market.History.Cataloger do defp compile_records(state) do price_records = state.price_records + secondary_coin_price_records = state |> Map.get(:secondary_coin_price_records, []) market_cap_records = state.market_cap_records tvl_records = state.tvl_records - all_records = price_records ++ market_cap_records ++ tvl_records + all_records = price_records ++ market_cap_records ++ tvl_records ++ secondary_coin_price_records all_records - |> Enum.group_by(fn %{date: date} -> date end) + |> Enum.group_by(fn %{date: date} = value -> {date, Map.get(value, :secondary_coin, false)} end) |> Map.values() |> Enum.map(fn a -> Enum.reduce(a, %{}, fn x, acc -> Map.merge(x, acc) end) diff --git a/apps/explorer/lib/explorer/market/history/source/price.ex b/apps/explorer/lib/explorer/market/history/source/price.ex index 07924c1fca..c8d697fa47 100644 --- a/apps/explorer/lib/explorer/market/history/source/price.ex +++ b/apps/explorer/lib/explorer/market/history/source/price.ex @@ -9,11 +9,13 @@ defmodule Explorer.Market.History.Source.Price do @type record :: %{ closing_price: Decimal.t(), date: Date.t(), - opening_price: Decimal.t() + opening_price: Decimal.t(), + secondary_coin: boolean() } @doc """ Fetch history for a specified amount of days in the past. """ - @callback fetch_price_history(previous_days :: non_neg_integer()) :: {:ok, [record()]} | :error + @callback fetch_price_history(previous_days :: non_neg_integer(), secondary_coin :: boolean()) :: + {:ok, [record()]} | :error end diff --git a/apps/explorer/lib/explorer/market/history/source/price/coin_gecko.ex b/apps/explorer/lib/explorer/market/history/source/price/coin_gecko.ex index ef49d1f19c..8cd7ca2203 100644 --- a/apps/explorer/lib/explorer/market/history/source/price/coin_gecko.ex +++ b/apps/explorer/lib/explorer/market/history/source/price/coin_gecko.ex @@ -11,14 +11,14 @@ defmodule Explorer.Market.History.Source.Price.CoinGecko do @behaviour SourcePrice @impl SourcePrice - def fetch_price_history(previous_days) do - url = ExchangeRatesSourceCoinGecko.history_url(previous_days) + def fetch_price_history(previous_days, secondary_coin? \\ false) do + url = ExchangeRatesSourceCoinGecko.history_url(previous_days, secondary_coin?) case Source.http_request(url, ExchangeRatesSourceCoinGecko.headers()) do {:ok, data} -> result = data - |> format_data() + |> format_data(secondary_coin?) {:ok, result} @@ -27,10 +27,10 @@ defmodule Explorer.Market.History.Source.Price.CoinGecko do end end - @spec format_data(term()) :: SourcePrice.record() | nil - defp format_data(nil), do: nil + @spec format_data(term(), boolean()) :: SourcePrice.record() | nil + defp format_data(nil, _), do: nil - defp format_data(data) do + defp format_data(data, secondary_coin?) do prices = data["prices"] for [date, price] <- prices do @@ -39,7 +39,8 @@ defmodule Explorer.Market.History.Source.Price.CoinGecko do %{ closing_price: Decimal.new(to_string(price)), date: CryptoCompare.date(date), - opening_price: Decimal.new(to_string(price)) + opening_price: Decimal.new(to_string(price)), + secondary_coin: secondary_coin? } end end diff --git a/apps/explorer/lib/explorer/market/history/source/price/coin_market_cap.ex b/apps/explorer/lib/explorer/market/history/source/price/coin_market_cap.ex index 0a8c4bf28d..c226edfcdd 100644 --- a/apps/explorer/lib/explorer/market/history/source/price/coin_market_cap.ex +++ b/apps/explorer/lib/explorer/market/history/source/price/coin_market_cap.ex @@ -10,15 +10,18 @@ defmodule Explorer.Market.History.Source.Price.CoinMarketCap do @behaviour SourcePrice @impl SourcePrice - def fetch_price_history(_previous_days \\ nil) do - url = ExchangeRatesSourceCoinMarketCap.source_url() + def fetch_price_history(_previous_days \\ nil, secondary_coin? \\ false) do + url = + if secondary_coin?, + do: ExchangeRatesSourceCoinMarketCap.source_url(:secondary_coin), + else: ExchangeRatesSourceCoinMarketCap.source_url() if url do case Source.http_request(url, ExchangeRatesSourceCoinMarketCap.headers()) do {:ok, data} -> result = data - |> format_data() + |> format_data(secondary_coin?) {:ok, result} @@ -30,10 +33,10 @@ defmodule Explorer.Market.History.Source.Price.CoinMarketCap do end end - @spec format_data(term()) :: SourcePrice.record() | nil - defp format_data(nil), do: nil + @spec format_data(term(), boolean()) :: SourcePrice.record() | nil + defp format_data(nil, _), do: nil - defp format_data(%{"data" => _} = json_data) do + defp format_data(%{"data" => _} = json_data, secondary_coin?) do market_data = json_data["data"] token_properties = ExchangeRatesSourceCoinMarketCap.get_token_properties(market_data) @@ -48,7 +51,8 @@ defmodule Explorer.Market.History.Source.Price.CoinMarketCap do %{ closing_price: current_price_usd, date: last_updated, - opening_price: current_price_usd + opening_price: current_price_usd, + secondary_coin: secondary_coin? } ] end diff --git a/apps/explorer/lib/explorer/market/history/source/price/crypto_compare.ex b/apps/explorer/lib/explorer/market/history/source/price/crypto_compare.ex index e1b31f03cc..297f723480 100644 --- a/apps/explorer/lib/explorer/market/history/source/price/crypto_compare.ex +++ b/apps/explorer/lib/explorer/market/history/source/price/crypto_compare.ex @@ -18,15 +18,15 @@ defmodule Explorer.Market.History.Source.Price.CryptoCompare do @typep unix_timestamp :: non_neg_integer() @impl SourcePrice - def fetch_price_history(previous_days) do - url = history_url(previous_days) + def fetch_price_history(previous_days, secondary_coin?) do + url = history_url(previous_days, secondary_coin?) headers = [{"Content-Type", "application/json"}] case HTTPoison.get(url, headers) do {:ok, %Response{body: body, status_code: 200}} -> result = body - |> format_data() + |> format_data(secondary_coin?) |> reject_zeros() {:ok, result} @@ -49,23 +49,26 @@ defmodule Explorer.Market.History.Source.Price.CryptoCompare do |> DateTime.to_date() end - @spec format_data(String.t()) :: [SourcePrice.record()] - defp format_data(data) do + @spec format_data(String.t(), boolean()) :: [SourcePrice.record()] + defp format_data(data, secondary_coin?) do json = Jason.decode!(data) for item <- json["Data"] do %{ closing_price: Decimal.new(to_string(item["close"])), date: date(item["time"]), - opening_price: Decimal.new(to_string(item["open"])) + opening_price: Decimal.new(to_string(item["open"])), + secondary_coin: secondary_coin? } end end - @spec history_url(non_neg_integer()) :: String.t() - defp history_url(previous_days) do + @spec history_url(non_neg_integer(), boolean()) :: String.t() + defp history_url(previous_days, secondary_coin?) do + fsym = if secondary_coin?, do: config(:secondary_coin_symbol), else: Explorer.coin() + query_params = %{ - "fsym" => Explorer.coin(), + "fsym" => fsym, "limit" => previous_days, "tsym" => "USD" } @@ -78,4 +81,9 @@ defmodule Explorer.Market.History.Source.Price.CryptoCompare do Decimal.equal?(item.closing_price, 0) && Decimal.equal?(item.opening_price, 0) end) end + + @spec config(atom()) :: term + defp config(key) do + Application.get_env(:explorer, __MODULE__, [])[key] + end end diff --git a/apps/explorer/lib/explorer/market/market.ex b/apps/explorer/lib/explorer/market/market.ex index a65b643fa5..11edb9bc39 100644 --- a/apps/explorer/lib/explorer/market/market.ex +++ b/apps/explorer/lib/explorer/market/market.ex @@ -14,9 +14,9 @@ defmodule Explorer.Market do Today's date is include as part of the day count """ - @spec fetch_recent_history() :: [MarketHistory.t()] - def fetch_recent_history do - MarketHistoryCache.fetch() + @spec fetch_recent_history(boolean()) :: [MarketHistory.t()] + def fetch_recent_history(secondary_coin? \\ false) do + MarketHistoryCache.fetch(secondary_coin?) end @doc """ @@ -72,7 +72,7 @@ defmodule Explorer.Market do Repo.insert_all(MarketHistory, records_without_zeroes, on_conflict: market_history_on_conflict(), - conflict_target: [:date] + conflict_target: [:date, :secondary_coin] ) end diff --git a/apps/explorer/lib/explorer/market/market_history.ex b/apps/explorer/lib/explorer/market/market_history.ex index 6aa24f2f62..d8ddc3ad5a 100644 --- a/apps/explorer/lib/explorer/market/market_history.ex +++ b/apps/explorer/lib/explorer/market/market_history.ex @@ -20,5 +20,6 @@ defmodule Explorer.Market.MarketHistory do field(:opening_price, :decimal) field(:market_cap, :decimal) field(:tvl, :decimal) + field(:secondary_coin, :boolean) end end diff --git a/apps/explorer/lib/explorer/market/market_history_cache.ex b/apps/explorer/lib/explorer/market/market_history_cache.ex index 81307bc34f..8d095ab359 100644 --- a/apps/explorer/lib/explorer/market/market_history_cache.ex +++ b/apps/explorer/lib/explorer/market/market_history_cache.ex @@ -15,12 +15,15 @@ defmodule Explorer.Market.MarketHistoryCache do # 6 hours @recent_days 30 - def fetch do - if cache_expired?(@last_update_key) do + def fetch(secondary_coin? \\ false) do + @last_update_key + |> cache_expired?() + |> if do update_cache() else fetch_from_cache(@history_key) end + |> Enum.filter(&(&1.secondary_coin == secondary_coin?)) end def cache_name, do: @cache_name diff --git a/apps/explorer/priv/repo/migrations/20240219143204_add_volume_24h_to_tokens.exs b/apps/explorer/priv/repo/migrations/20240219143204_add_volume_24h_to_tokens.exs new file mode 100644 index 0000000000..cf3c9ad7b3 --- /dev/null +++ b/apps/explorer/priv/repo/migrations/20240219143204_add_volume_24h_to_tokens.exs @@ -0,0 +1,9 @@ +defmodule Explorer.Repo.Migrations.AddVolume24hToTokens do + use Ecto.Migration + + def change do + alter table(:tokens) do + add(:volume_24h, :decimal) + end + end +end diff --git a/apps/explorer/priv/repo/migrations/20240226151331_add_secondary_coin_market_history.exs b/apps/explorer/priv/repo/migrations/20240226151331_add_secondary_coin_market_history.exs new file mode 100644 index 0000000000..799cf62ab0 --- /dev/null +++ b/apps/explorer/priv/repo/migrations/20240226151331_add_secondary_coin_market_history.exs @@ -0,0 +1,12 @@ +defmodule Explorer.Repo.Migrations.AddSecondaryCoinMarketHistory do + use Ecto.Migration + + def change do + alter table(:market_history) do + add(:secondary_coin, :boolean, default: false) + end + + drop_if_exists(unique_index(:market_history, [:date])) + create(unique_index(:market_history, [:date, :secondary_coin])) + end +end diff --git a/apps/explorer/test/explorer/counters/fresh_pending_transactions_counter_test.exs b/apps/explorer/test/explorer/counters/fresh_pending_transactions_counter_test.exs new file mode 100644 index 0000000000..6a93612e04 --- /dev/null +++ b/apps/explorer/test/explorer/counters/fresh_pending_transactions_counter_test.exs @@ -0,0 +1,27 @@ +defmodule Explorer.Counters.FreshPendingTransactionsCounterTest do + use Explorer.DataCase + + alias Explorer.Counters.FreshPendingTransactionsCounter + + test "populates the cache with the number of pending transactions addresses" do + insert(:transaction) + insert(:transaction) + insert(:transaction) + + start_supervised!(FreshPendingTransactionsCounter) + FreshPendingTransactionsCounter.consolidate() + + assert FreshPendingTransactionsCounter.fetch([]) == Decimal.new("3") + end + + test "count only fresh transactions" do + insert(:transaction, inserted_at: Timex.shift(Timex.now(), hours: -2)) + insert(:transaction) + insert(:transaction) + + start_supervised!(FreshPendingTransactionsCounter) + FreshPendingTransactionsCounter.consolidate() + + assert FreshPendingTransactionsCounter.fetch([]) == Decimal.new("2") + end +end diff --git a/apps/explorer/test/explorer/counters/transactions_24h_stats_test.exs b/apps/explorer/test/explorer/counters/transactions_24h_stats_test.exs new file mode 100644 index 0000000000..96a5fccaff --- /dev/null +++ b/apps/explorer/test/explorer/counters/transactions_24h_stats_test.exs @@ -0,0 +1,61 @@ +defmodule Explorer.Counters.Transactions24hStatsTest do + use Explorer.DataCase + + alias Explorer.Counters.Transactions24hStats + + test "populates the cache with transaction counters" do + block = insert(:block, base_fee_per_gas: 50) + address = insert(:address) + + # fee = 10000 + + insert(:transaction, + from_address: address, + block: block, + block_number: block.number, + cumulative_gas_used: 0, + index: 0, + gas_price: 100, + gas_used: 100 + ) + + # fee = 15000 + + insert(:transaction, + from_address: address, + block: block, + block_number: block.number, + cumulative_gas_used: 100, + index: 1, + gas_price: 150, + gas_used: 100, + max_priority_fee_per_gas: 100, + max_fee_per_gas: 200 + ) + + # fee = 10000 + + insert(:transaction, + from_address: address, + block: block, + block_number: block.number, + cumulative_gas_used: 200, + index: 2, + gas_price: 100, + gas_used: 100, + max_priority_fee_per_gas: 70, + max_fee_per_gas: 100 + ) + + start_supervised!(Transactions24hStats) + Transactions24hStats.consolidate() + + transaction_count = Transactions24hStats.fetch_count([]) + transaction_fee_sum = Transactions24hStats.fetch_fee_sum([]) + transaction_fee_average = Transactions24hStats.fetch_fee_average([]) + + assert transaction_count == Decimal.new("3") + assert transaction_fee_sum == Decimal.new("35000") + assert transaction_fee_average == Decimal.new("11667") + end +end diff --git a/apps/explorer/test/explorer/exchange_rates/source/coin_gecko_test.exs b/apps/explorer/test/explorer/exchange_rates/source/coin_gecko_test.exs index e16d056422..aa17f90401 100644 --- a/apps/explorer/test/explorer/exchange_rates/source/coin_gecko_test.exs +++ b/apps/explorer/test/explorer/exchange_rates/source/coin_gecko_test.exs @@ -71,7 +71,7 @@ defmodule Explorer.ExchangeRates.Source.CoinGeckoTest do end test "composes cg url to list of contract address hashes" do - assert "https://api.coingecko.com/api/v3/simple/token_price/ethereum?vs_currencies=usd&include_market_cap=true&contract_addresses=0xdAC17F958D2ee523a2206206994597C13D831ec7" == + assert "https://api.coingecko.com/api/v3/simple/token_price/ethereum?vs_currencies=usd&include_market_cap=true&include_24hr_vol=true&contract_addresses=0xdAC17F958D2ee523a2206206994597C13D831ec7" == CoinGecko.source_url(["0xdAC17F958D2ee523a2206206994597C13D831ec7"]) end diff --git a/apps/explorer/test/explorer/exchange_rates/token_exchange_rates_test.exs b/apps/explorer/test/explorer/exchange_rates/token_exchange_rates_test.exs index 2e5f9ee1a6..a6d3255a80 100644 --- a/apps/explorer/test/explorer/exchange_rates/token_exchange_rates_test.exs +++ b/apps/explorer/test/explorer/exchange_rates/token_exchange_rates_test.exs @@ -77,7 +77,9 @@ defmodule Explorer.TokenExchangeRatesTest do "GET", "/simple/token_price/ethereum", fn conn -> - assert conn.query_string == "vs_currencies=usd&include_market_cap=true&contract_addresses=#{joined_addresses}" + assert conn.query_string == + "vs_currencies=usd&include_market_cap=true&include_24hr_vol=true&contract_addresses=#{joined_addresses}" + Conn.resp(conn, 200, Jason.encode!(token_exchange_rates)) end ) @@ -159,7 +161,9 @@ defmodule Explorer.TokenExchangeRatesTest do "GET", "/simple/token_price/ethereum", fn conn -> - assert conn.query_string == "vs_currencies=usd&include_market_cap=true&contract_addresses=#{joined_addresses}" + assert conn.query_string == + "vs_currencies=usd&include_market_cap=true&include_24hr_vol=true&contract_addresses=#{joined_addresses}" + Conn.resp(conn, 200, "{}") end ) @@ -239,7 +243,9 @@ defmodule Explorer.TokenExchangeRatesTest do "GET", "/simple/token_price/ethereum", fn conn -> - assert conn.query_string == "vs_currencies=usd&include_market_cap=true&contract_addresses=#{joined_addresses}" + assert conn.query_string == + "vs_currencies=usd&include_market_cap=true&include_24hr_vol=true&contract_addresses=#{joined_addresses}" + Conn.resp(conn, 429, "Too many requests") end ) diff --git a/apps/explorer/test/explorer/market/history/cataloger_test.exs b/apps/explorer/test/explorer/market/history/cataloger_test.exs index f7a86a960e..8e784ed0dc 100644 --- a/apps/explorer/test/explorer/market/history/cataloger_test.exs +++ b/apps/explorer/test/explorer/market/history/cataloger_test.exs @@ -54,13 +54,17 @@ defmodule Explorer.Market.History.CatalogerTest do """ Bypass.expect(bypass, fn conn -> Conn.resp(conn, 200, resp) end) - records = [%{date: ~D[2018-04-01], closing_price: Decimal.new(10), opening_price: Decimal.new(5)}] - expect(TestSource, :fetch_price_history, fn 1 -> {:ok, records} end) + + records = [ + %{date: ~D[2018-04-01], closing_price: Decimal.new(10), opening_price: Decimal.new(5), secondary_coin: false} + ] + + expect(TestSource, :fetch_price_history, fn 1, _ -> {:ok, records} end) set_mox_global() state = %{} assert {:noreply, state} == Cataloger.handle_info({:fetch_price_history, 1}, state) - assert_receive {_ref, {:price_history, {1, 0, {:ok, ^records}}}} + assert_receive {_ref, {:price_history, {1, 0, false, {:ok, ^records}}}} end test "handle_info with successful tasks (price, market cap and tvl)" do @@ -80,15 +84,15 @@ defmodule Explorer.Market.History.CatalogerTest do state2 = Map.put(state, :market_cap_records, market_cap_records) - state3 = Map.put(state2, :tvl_records, tvl_records) + assert {:noreply, state} == + Cataloger.handle_info({nil, {:price_history, {1, 0, false, {:ok, price_records}}}}, state) - assert {:noreply, state} == Cataloger.handle_info({nil, {:price_history, {1, 0, {:ok, price_records}}}}, state) - assert_receive {:fetch_market_cap_history, 365} + assert_receive {:fetch_market_cap_history, 1} assert {:noreply, state2} == Cataloger.handle_info({nil, {:market_cap_history, {0, 3, {:ok, market_cap_records}}}}, state) - assert {:noreply, state3} == + assert {:noreply, %{}} == Cataloger.handle_info({nil, {:tvl_history, {0, 3, {:ok, tvl_records}}}}, state2) assert record2 = Repo.get_by(MarketHistory, date: Enum.at(price_records, 1).date) @@ -113,15 +117,15 @@ defmodule Explorer.Market.History.CatalogerTest do state2 = Map.put(state, :market_cap_records, market_cap_records) - state3 = Map.put(state2, :tvl_records, []) + assert {:noreply, state} == + Cataloger.handle_info({nil, {:price_history, {1, 0, false, {:ok, price_records}}}}, state) - assert {:noreply, state} == Cataloger.handle_info({nil, {:price_history, {1, 0, {:ok, price_records}}}}, state) - assert_receive {:fetch_market_cap_history, 365} + assert_receive {:fetch_market_cap_history, 1} assert {:noreply, state2} == Cataloger.handle_info({nil, {:market_cap_history, {0, 3, {:ok, market_cap_records}}}}, state) - assert {:noreply, state3} == + assert {:noreply, %{}} == Cataloger.handle_info({nil, {:tvl_history, {0, 3, {:ok, tvl_records}}}}, state2) assert record = Repo.get_by(MarketHistory, date: Enum.at(price_records, 0).date) @@ -142,15 +146,15 @@ defmodule Explorer.Market.History.CatalogerTest do state2 = Map.put(state, :market_cap_records, market_cap_records) - state3 = Map.put(state2, :tvl_records, tvl_records) + assert {:noreply, state} == + Cataloger.handle_info({nil, {:price_history, {1, 0, false, {:ok, price_records}}}}, state) - assert {:noreply, state} == Cataloger.handle_info({nil, {:price_history, {1, 0, {:ok, price_records}}}}, state) - assert_receive {:fetch_market_cap_history, 365} + assert_receive {:fetch_market_cap_history, 1} assert {:noreply, state2} == Cataloger.handle_info({nil, {:market_cap_history, {0, 3, {:ok, market_cap_records}}}}, state) - assert {:noreply, state3} == + assert {:noreply, %{}} == Cataloger.handle_info({nil, {:tvl_history, {0, 3, {:ok, tvl_records}}}}, state2) assert record = Repo.get_by(MarketHistory, date: Enum.at(price_records, 0).date) @@ -159,6 +163,72 @@ defmodule Explorer.Market.History.CatalogerTest do assert record.tvl == nil end + test "current day values are saved in state" do + bypass = Bypass.open() + Application.put_env(:explorer, CryptoCompare, base_url: "http://localhost:#{bypass.port}") + old_env = Application.get_all_env(:explorer) + + Application.put_env(:explorer, Explorer.History.Process, base_backoff: 0) + + resp = + &""" + { + "Response": "Success", + "Type": 100, + "Aggregated": false, + "TimeTo": 1522569618, + "TimeFrom": 1522566018, + "FirstValueInArray": true, + "ConversionType": { + "type": "multiply", + "conversionSymbol": "ETH" + }, + "Data": [{ + "time": #{&1}, + "high": 10, + "low": 5, + "open": 5, + "volumefrom": 0, + "volumeto": 0, + "close": #{&2}, + "conversionType": "multiply", + "conversionSymbol": "ETH" + }], + "RateLimit": {}, + "HasWarning": false + } + """ + + Bypass.expect(bypass, fn conn -> + case conn.params["limit"] do + "365" -> Conn.resp(conn, 200, resp.(1_522_566_018, 10)) + _ -> Conn.resp(conn, 200, resp.(1_522_633_818, 20)) + end + end) + + {:ok, pid} = Cataloger.start_link([]) + + :timer.sleep(4000) + + Process.send(pid, {:fetch_price_history, 1}, []) + + :timer.sleep(4000) + + assert [ + %Explorer.Market.MarketHistory{ + date: ~D[2018-04-01] + } = first_entry, + %Explorer.Market.MarketHistory{ + date: ~D[2018-04-02] + } = second_entry + ] = MarketHistory |> Repo.all() + + assert Decimal.eq?(first_entry.closing_price, Decimal.new(10)) + assert Decimal.eq?(second_entry.closing_price, Decimal.new(20)) + + Application.put_all_env(explorer: old_env) + end + test "handle info for DOWN message" do assert {:noreply, %{}} == Cataloger.handle_info({:DOWN, nil, :process, nil, nil}, %{}) end diff --git a/apps/explorer/test/explorer/market/history/source/price/crypto_compare_test.exs b/apps/explorer/test/explorer/market/history/source/price/crypto_compare_test.exs index 2d8311b0ae..82f60c12d9 100644 --- a/apps/explorer/test/explorer/market/history/source/price/crypto_compare_test.exs +++ b/apps/explorer/test/explorer/market/history/source/price/crypto_compare_test.exs @@ -63,28 +63,31 @@ defmodule Explorer.Market.History.Source.Price.CryptoCompareTest do %{ closing_price: Decimal.from_float(9655.77), date: ~D[2018-04-24], - opening_price: Decimal.from_float(8967.86) + opening_price: Decimal.from_float(8967.86), + secondary_coin: false }, %{ closing_price: Decimal.from_float(8873.62), date: ~D[2018-04-25], - opening_price: Decimal.from_float(9657.69) + opening_price: Decimal.from_float(9657.69), + secondary_coin: false }, %{ closing_price: Decimal.from_float(8804.32), date: ~D[2018-04-26], - opening_price: Decimal.from_float(8873.57) + opening_price: Decimal.from_float(8873.57), + secondary_coin: false } ] - assert {:ok, expected} == CryptoCompare.fetch_price_history(3) + assert {:ok, expected} == CryptoCompare.fetch_price_history(3, false) end test "with errored request", %{bypass: bypass} do error_text = ~S({"error": "server error"}) Bypass.expect(bypass, fn conn -> Conn.resp(conn, 500, error_text) end) - assert :error == CryptoCompare.fetch_price_history(3) + assert :error == CryptoCompare.fetch_price_history(3, false) end test "rejects empty prices", %{bypass: bypass} do @@ -135,10 +138,15 @@ defmodule Explorer.Market.History.Source.Price.CryptoCompareTest do Bypass.expect(bypass, fn conn -> Conn.resp(conn, 200, json) end) expected = [ - %{closing_price: Decimal.from_float(8804.32), date: ~D[2018-04-26], opening_price: Decimal.from_float(8873.57)} + %{ + closing_price: Decimal.from_float(8804.32), + date: ~D[2018-04-26], + opening_price: Decimal.from_float(8873.57), + secondary_coin: false + } ] - assert {:ok, expected} == CryptoCompare.fetch_price_history(3) + assert {:ok, expected} == CryptoCompare.fetch_price_history(3, false) end end end diff --git a/apps/explorer/test/support/fakes/no_op_price_source.ex b/apps/explorer/test/support/fakes/no_op_price_source.ex index b9a460ac88..fa896d9003 100644 --- a/apps/explorer/test/support/fakes/no_op_price_source.ex +++ b/apps/explorer/test/support/fakes/no_op_price_source.ex @@ -6,7 +6,7 @@ defmodule Explorer.ExchangeRates.Source.NoOpPriceSource do @behaviour SourcePrice @impl SourcePrice - def fetch_price_history(_previous_days) do + def fetch_price_history(_previous_days, _secondary_coin?) do {:ok, []} end end diff --git a/config/config_helper.exs b/config/config_helper.exs index b67e23e0e4..a703ac522b 100644 --- a/config/config_helper.exs +++ b/config/config_helper.exs @@ -170,6 +170,20 @@ defmodule ConfigHelper do end end + @spec exchange_rates_secondary_coin_price_source() :: Price.CoinGecko | Price.CoinMarketCap | Price.CryptoCompare + def exchange_rates_secondary_coin_price_source do + cmc_secondary_coin_id = System.get_env("EXCHANGE_RATES_COINMARKETCAP_SECONDARY_COIN_ID") + cg_secondary_coin_id = System.get_env("EXCHANGE_RATES_COINGECKO_SECONDARY_COIN_ID") + cc_secondary_coin_symbol = System.get_env("EXCHANGE_RATES_CRYPTOCOMPARE_SECONDARY_COIN_SYMBOL") + + cond do + cg_secondary_coin_id && cg_secondary_coin_id !== "" -> Price.CoinGecko + cmc_secondary_coin_id && cmc_secondary_coin_id !== "" -> Price.CoinMarketCap + cc_secondary_coin_symbol && cc_secondary_coin_symbol !== "" -> Price.CryptoCompare + true -> Price.CryptoCompare + end + end + def block_transformer do block_transformers = %{ "clique" => Blocks.Clique, diff --git a/config/runtime.exs b/config/runtime.exs index 96e2938e44..b1f384bdba 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -292,10 +292,20 @@ config :explorer, Explorer.Counters.AddressTokenTransfersCounter, cache_period: ConfigHelper.parse_time_env_var("CACHE_ADDRESS_TOKEN_TRANSFERS_COUNTER_PERIOD", "1h") config :explorer, Explorer.Counters.LastOutputRootSizeCounter, - enabled: true, - enable_consolidation: true, + enabled: ConfigHelper.chain_type() == "optimism", + enable_consolidation: ConfigHelper.chain_type() == "optimism", cache_period: ConfigHelper.parse_time_env_var("CACHE_OPTIMISM_LAST_OUTPUT_ROOT_SIZE_COUNTER_PERIOD", "5m") +config :explorer, Explorer.Counters.Transactions24hStats, + enabled: true, + cache_period: ConfigHelper.parse_time_env_var("CACHE_TRANSACTIONS_24H_STATS_PERIOD", "1h"), + enable_consolidation: true + +config :explorer, Explorer.Counters.FreshPendingTransactionsCounter, + enabled: true, + cache_period: ConfigHelper.parse_time_env_var("CACHE_FRESH_PENDING_TRANSACTIONS_COUNTER_PERIOD", "5m"), + enable_consolidation: true + config :explorer, Explorer.ExchangeRates, store: :ets, enabled: !disable_exchange_rates?, @@ -304,20 +314,31 @@ config :explorer, Explorer.ExchangeRates, config :explorer, Explorer.ExchangeRates.Source, source: ConfigHelper.exchange_rates_source(), price_source: ConfigHelper.exchange_rates_price_source(), + secondary_coin_price_source: ConfigHelper.exchange_rates_secondary_coin_price_source(), market_cap_source: ConfigHelper.exchange_rates_market_cap_source(), tvl_source: ConfigHelper.exchange_rates_tvl_source() +cmc_secondary_coin_id = System.get_env("EXCHANGE_RATES_COINMARKETCAP_SECONDARY_COIN_ID") + config :explorer, Explorer.ExchangeRates.Source.CoinMarketCap, api_key: System.get_env("EXCHANGE_RATES_COINMARKETCAP_API_KEY"), - coin_id: System.get_env("EXCHANGE_RATES_COINMARKETCAP_COIN_ID") + coin_id: System.get_env("EXCHANGE_RATES_COINMARKETCAP_COIN_ID"), + secondary_coin_id: cmc_secondary_coin_id + +cg_secondary_coin_id = System.get_env("EXCHANGE_RATES_COINGECKO_SECONDARY_COIN_ID") config :explorer, Explorer.ExchangeRates.Source.CoinGecko, platform: System.get_env("EXCHANGE_RATES_COINGECKO_PLATFORM_ID"), api_key: System.get_env("EXCHANGE_RATES_COINGECKO_API_KEY"), - coin_id: System.get_env("EXCHANGE_RATES_COINGECKO_COIN_ID") + coin_id: System.get_env("EXCHANGE_RATES_COINGECKO_COIN_ID"), + secondary_coin_id: cg_secondary_coin_id config :explorer, Explorer.ExchangeRates.Source.DefiLlama, coin_id: System.get_env("EXCHANGE_RATES_DEFILLAMA_COIN_ID") +cc_secondary_coin_symbol = System.get_env("EXCHANGE_RATES_CRYPTOCOMPARE_SECONDARY_COIN_SYMBOL") + +config :explorer, Explorer.Market.History.Source.Price.CryptoCompare, secondary_coin_symbol: cc_secondary_coin_symbol + config :explorer, Explorer.ExchangeRates.TokenExchangeRates, enabled: !ConfigHelper.parse_bool_env_var("DISABLE_TOKEN_EXCHANGE_RATE", "true"), interval: ConfigHelper.parse_time_env_var("TOKEN_EXCHANGE_RATE_INTERVAL", "5s"), @@ -326,7 +347,8 @@ config :explorer, Explorer.ExchangeRates.TokenExchangeRates, config :explorer, Explorer.Market.History.Cataloger, enabled: !disable_indexer? && !disable_exchange_rates?, - history_fetch_interval: ConfigHelper.parse_time_env_var("MARKET_HISTORY_FETCH_INTERVAL", "1h") + history_fetch_interval: ConfigHelper.parse_time_env_var("MARKET_HISTORY_FETCH_INTERVAL", "1h"), + secondary_coin_enabled: cmc_secondary_coin_id || cg_secondary_coin_id || cc_secondary_coin_symbol config :explorer, Explorer.Chain.Transaction, suave_bid_contracts: System.get_env("SUAVE_BID_CONTRACTS", "") diff --git a/config/runtime/test.exs b/config/runtime/test.exs index ca3eed98e1..786755f812 100644 --- a/config/runtime/test.exs +++ b/config/runtime/test.exs @@ -16,6 +16,10 @@ config :block_scout_web, BlockScoutWeb.API.V2, enabled: true ### Explorer ### ################ +config :explorer, Explorer.Counters.Transactions24hStats, + cache_period: ConfigHelper.parse_time_env_var("CACHE_TRANSACTIONS_24H_STATS_PERIOD", "1h"), + enable_consolidation: false + variant = Variant.get() Code.require_file("#{variant}.exs", "apps/explorer/config/test") diff --git a/cspell.json b/cspell.json index cad6b4ad11..ec008f2588 100644 --- a/cspell.json +++ b/cspell.json @@ -189,6 +189,7 @@ "cooldown", "cooltesthost", "crossorigin", + "CRYPTOCOMPARE", "ctbs", "ctid", "cumalative", @@ -244,6 +245,8 @@ "exvcr", "falala", "FEVM", + "filecoin", + "Filecoin", "Filesize", "Filecoin", "fkey", @@ -378,6 +381,7 @@ "noproc", "noreferrer", "noreply", + "NOTOK", "noves", "nowarn", "nowrap", @@ -501,6 +505,7 @@ "successa", "successb", "supernet", + "sushiswap", "swal", "sweetalert", "tabindex", @@ -562,6 +567,7 @@ "valuemin", "valuenow", "varint", + "verifyproxycontract", "verifysourcecode", "viewerjs", "volumefrom", @@ -588,6 +594,7 @@ "yellowgreen", "zaphod", "zeppelinos", + "zetachain", "zftv", "ziczr", "zindex", diff --git a/docker-compose/envs/common-blockscout.env b/docker-compose/envs/common-blockscout.env index abf1c40689..11d5e78725 100644 --- a/docker-compose/envs/common-blockscout.env +++ b/docker-compose/envs/common-blockscout.env @@ -41,9 +41,12 @@ EXCHANGE_RATES_COIN= # EXCHANGE_RATES_TVL_SOURCE= # EXCHANGE_RATES_PRICE_SOURCE= # EXCHANGE_RATES_COINGECKO_COIN_ID= +# EXCHANGE_RATES_COINGECKO_SECONDARY_COIN_ID= # EXCHANGE_RATES_COINGECKO_API_KEY= # EXCHANGE_RATES_COINMARKETCAP_API_KEY= # EXCHANGE_RATES_COINMARKETCAP_COIN_ID= +# EXCHANGE_RATES_COINMARKETCAP_SECONDARY_COIN_ID= +# EXCHANGE_RATES_CRYPTOCOMPARE_SECONDARY_COIN_SYMBOL= POOL_SIZE=80 # EXCHANGE_RATES_COINGECKO_PLATFORM_ID= # TOKEN_EXCHANGE_RATE_INTERVAL= @@ -91,6 +94,8 @@ CACHE_MARKET_HISTORY_PERIOD=21600 CACHE_ADDRESS_TRANSACTIONS_COUNTER_PERIOD=1800 CACHE_ADDRESS_TOKENS_USD_SUM_PERIOD=3600 CACHE_ADDRESS_TOKEN_TRANSFERS_COUNTER_PERIOD=1800 +# CACHE_TRANSACTIONS_24H_STATS_PERIOD= +# CACHE_FRESH_PENDING_TRANSACTIONS_COUNTER_PERIOD= TOKEN_METADATA_UPDATE_INTERVAL=172800 CONTRACT_VERIFICATION_ALLOWED_SOLIDITY_EVM_VERSIONS=homestead,tangerineWhistle,spuriousDragon,byzantium,constantinople,petersburg,istanbul,berlin,london,paris,shanghai,cancun,default CONTRACT_VERIFICATION_ALLOWED_VYPER_EVM_VERSIONS=byzantium,constantinople,petersburg,istanbul,berlin,paris,shanghai,cancun,default