From e485c1d7d959eca596b707619e79f213b6b9a371 Mon Sep 17 00:00:00 2001 From: Viktor Baranov Date: Tue, 3 Oct 2023 19:45:35 +0300 Subject: [PATCH] DefiLlama TVL source --- CHANGELOG.md | 1 + .../controllers/api/v2/stats_controller.ex | 2 +- .../chain/market_history_chart_controller.ex | 2 +- .../exchange_rates/source/defillama.ex | 46 +++++++++ .../lib/explorer/market/history/cataloger.ex | 95 ++++++++++++++++--- .../lib/explorer/market/history/source/tvl.ex | 18 ++++ .../market/history/source/tvl/defillama.ex | 49 ++++++++++ .../lib/explorer/market/market_history.ex | 5 +- ...093553_add_tvl_to_market_history_table.exs | 9 ++ .../market/history/cataloger_test.exs | 91 ++++++++++++++---- config/config_helper.exs | 10 +- config/runtime.exs | 5 +- cspell.json | 1 + docker-compose/envs/common-blockscout.env | 1 + 14 files changed, 299 insertions(+), 36 deletions(-) create mode 100644 apps/explorer/lib/explorer/exchange_rates/source/defillama.ex create mode 100644 apps/explorer/lib/explorer/market/history/source/tvl.ex create mode 100644 apps/explorer/lib/explorer/market/history/source/tvl/defillama.ex create mode 100644 apps/explorer/priv/repo/migrations/20231003093553_add_tvl_to_market_history_table.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index 369e57cad4..6671ea021c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Features - [#8472](https://github.com/blockscout/blockscout/pull/8472) - Integrate `/api/v2/bytecodes/sources:search-all` of `eth_bytecode_db` +- [#8589](https://github.com/blockscout/blockscout/pull/8589) - DefiLlama TVL source - [#8544](https://github.com/blockscout/blockscout/pull/8544) - Fix `nil` `"structLogs"` - [#8561](https://github.com/blockscout/blockscout/pull/8561), [#8564](https://github.com/blockscout/blockscout/pull/8564) - Get historical market cap data from CoinGecko - [#8386](https://github.com/blockscout/blockscout/pull/8386) - Add `owner_address_hash` to the `token_instances` diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/stats_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/stats_controller.ex index 098b05f7e9..80408045cf 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/stats_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/stats_controller.ex @@ -112,7 +112,7 @@ defmodule BlockScoutWeb.API.V2.StatsController do data -> data end - |> Enum.map(fn day -> Map.take(day, [:closing_price, :market_cap, :date]) end) + |> Enum.map(fn day -> Map.take(day, [:closing_price, :market_cap, :tvl, :date]) end) market_history_data = MarketHistoryChartController.encode_market_history_data(price_history_data, current_total_supply) diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/chain/market_history_chart_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/chain/market_history_chart_controller.ex index a9816adcbd..194d865e22 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/chain/market_history_chart_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/chain/market_history_chart_controller.ex @@ -59,7 +59,7 @@ defmodule BlockScoutWeb.Chain.MarketHistoryChartController do day |> Map.put(:market_cap, market_cap) - |> Map.take([:closing_price, :market_cap, :date]) + |> Map.take([:closing_price, :market_cap, :tvl, :date]) end) |> Jason.encode() |> case do diff --git a/apps/explorer/lib/explorer/exchange_rates/source/defillama.ex b/apps/explorer/lib/explorer/exchange_rates/source/defillama.ex new file mode 100644 index 0000000000..e7cae25e02 --- /dev/null +++ b/apps/explorer/lib/explorer/exchange_rates/source/defillama.ex @@ -0,0 +1,46 @@ +defmodule Explorer.ExchangeRates.Source.DefiLlama do + @moduledoc """ + Adapter for fetching exchange rates from https://defillama.com/ + + """ + + alias Explorer.ExchangeRates.Source + + @behaviour Source + + @impl Source + def format_data(_), do: [] + + @spec history_url(non_neg_integer()) :: String.t() + def history_url(_previous_days) do + "#{base_url()}/historicalChainTvl" + end + + @impl Source + def source_url do + "" + end + + @impl Source + def source_url(_) do + "" + end + + @impl Source + def headers do + [] + end + + defp base_url do + base_free_url() + end + + defp base_free_url do + config(:base_url) || "https://api.llama.fi/v2" + end + + @spec config(atom()) :: term + defp config(key) do + Application.get_env(:explorer, __MODULE__, [])[key] + end +end diff --git a/apps/explorer/lib/explorer/market/history/cataloger.ex b/apps/explorer/lib/explorer/market/history/cataloger.ex index 9e93b7a9ce..f3c73de23d 100644 --- a/apps/explorer/lib/explorer/market/history/cataloger.ex +++ b/apps/explorer/lib/explorer/market/history/cataloger.ex @@ -19,6 +19,7 @@ defmodule Explorer.Market.History.Cataloger do @price_failed_attempts 10 @market_cap_failed_attempts 3 + @tvl_failed_attempts 3 @impl GenServer def init(:ok) do @@ -38,32 +39,61 @@ defmodule Explorer.Market.History.Cataloger do {:noreply, state} end + @impl GenServer + # Record fetch successful. + def handle_info({_ref, {:price_history, {_, _, {:ok, records}}}}, state) do + Process.send(self(), {:fetch_market_cap_history, 365}, []) + state = state |> Map.put_new(:price_records, records) + + {:noreply, state} + end + @impl GenServer def handle_info({:fetch_market_cap_history, day_count}, state) do fetch_market_cap_history(day_count) + state = state |> Map.put_new(:price_records, []) {:noreply, state} end @impl GenServer # Record fetch successful. - def handle_info({_ref, {:price_history, {_, _, {:ok, records}}}}, state) do - Process.send(self(), {:fetch_market_cap_history, 365}, []) - state = state |> Map.put_new(:price_records, records) + def handle_info({_ref, {:market_cap_history, {_, _, {:ok, nil}}}}, state) do + Process.send(self(), {:fetch_tvl_history, 365}, []) + state = state |> Map.put_new(:market_cap_records, []) - {:noreply, state |> Map.put_new(:price_records, state)} + {:noreply, state} end @impl GenServer # Record fetch successful. - def handle_info({_ref, {:market_cap_history, {_, _, {:ok, nil}}}}, state) do - market_cap_history(state.price_records, state) + def handle_info({_ref, {:market_cap_history, {_, _, {:ok, market_cap_records}}}}, state) do + Process.send(self(), {:fetch_tvl_history, 365}, []) + state = state |> Map.put_new(:market_cap_records, market_cap_records) + + {:noreply, state} + end + + @impl GenServer + def handle_info({:fetch_tvl_history, day_count}, state) do + fetch_tvl_history(day_count) + + {:noreply, state} end @impl GenServer # Record fetch successful. - def handle_info({_ref, {:market_cap_history, {_, _, {:ok, market_cap_records}}}}, state) do - records = compile_records(state.price_records, market_cap_records) + def handle_info({_ref, {:tvl_history, {_, _, {:ok, nil}}}}, state) do + state = state |> Map.put_new(:tvl_records, []) + records = compile_records(state) + market_cap_history(records, state) + end + + @impl GenServer + # Record fetch successful. + def handle_info({_ref, {:tvl_history, {_, _, {:ok, tvl_records}}}}, state) do + state = state |> Map.put_new(:tvl_records, tvl_records) + records = compile_records(state) market_cap_history(records, state) end @@ -87,6 +117,16 @@ defmodule Explorer.Market.History.Cataloger do {:noreply, state} end + # Failed to get records. Try again. + @impl GenServer + def handle_info({_ref, {:tvl_history, {day_count, failed_attempts, :error}}}, state) do + Logger.warn(fn -> "Failed to fetch market cap history. Trying again." end) + + fetch_tvl_history(day_count, failed_attempts + 1) + + {:noreply, state} + end + # Callback that a monitored process has shutdown. @impl GenServer def handle_info({:DOWN, _, :process, _, _}, state) do @@ -135,6 +175,15 @@ defmodule Explorer.Market.History.Cataloger do ) end + @spec source_tvl() :: module() + defp source_tvl do + config_or_default( + :tvl_source, + Explorer.ExchangeRates.Source, + Explorer.Market.History.Source.TVL.DefiLlama + ) + end + @spec fetch_price_history(non_neg_integer(), non_neg_integer()) :: Task.t() defp fetch_price_history(day_count, failed_attempts \\ 0) do Task.Supervisor.async_nolink(Explorer.MarketTaskSupervisor, fn -> @@ -161,11 +210,31 @@ defmodule Explorer.Market.History.Cataloger do end) end - defp compile_records(price_records, market_cap_records) do - price_records - |> Enum.zip(market_cap_records) - |> Enum.map(fn {price_map, market_cap_map} -> - Map.merge(price_map, market_cap_map) + @spec fetch_tvl_history(non_neg_integer()) :: Task.t() + defp fetch_tvl_history(day_count, failed_attempts \\ 0) do + Task.Supervisor.async_nolink(Explorer.MarketTaskSupervisor, fn -> + Process.sleep(HistoryProcess.delay(failed_attempts)) + + if failed_attempts < @tvl_failed_attempts do + {:tvl_history, {day_count, failed_attempts, source_tvl().fetch_tvl(day_count)}} + else + {:tvl_history, {day_count, failed_attempts, {:ok, nil}}} + end + end) + end + + defp compile_records(state) do + price_records = state.price_records + market_cap_records = state.market_cap_records + tvl_records = state.tvl_records + + all_records = price_records ++ market_cap_records ++ tvl_records + + all_records + |> Enum.group_by(fn %{date: date} -> date end) + |> Map.values() + |> Enum.map(fn a -> + Enum.reduce(a, %{}, fn x, acc -> Map.merge(x, acc) end) end) end end diff --git a/apps/explorer/lib/explorer/market/history/source/tvl.ex b/apps/explorer/lib/explorer/market/history/source/tvl.ex new file mode 100644 index 0000000000..19e7da498e --- /dev/null +++ b/apps/explorer/lib/explorer/market/history/source/tvl.ex @@ -0,0 +1,18 @@ +defmodule Explorer.Market.History.Source.TVL do + @moduledoc """ + Interface for a source that allows for fetching of TVL history. + """ + + @typedoc """ + Record of market values for a specific date. + """ + @type record :: %{ + date: Date.t(), + tvl: Decimal.t() + } + + @doc """ + Fetch history for a specified amount of days in the past. + """ + @callback fetch_tvl(previous_days :: non_neg_integer()) :: {:ok, [record()]} | :error +end diff --git a/apps/explorer/lib/explorer/market/history/source/tvl/defillama.ex b/apps/explorer/lib/explorer/market/history/source/tvl/defillama.ex new file mode 100644 index 0000000000..9fcf29186d --- /dev/null +++ b/apps/explorer/lib/explorer/market/history/source/tvl/defillama.ex @@ -0,0 +1,49 @@ +defmodule Explorer.Market.History.Source.TVL.DefiLlama do + @moduledoc """ + Adapter for fetching current market from DefiLlama. + """ + + alias Explorer.ExchangeRates.Source + alias Explorer.ExchangeRates.Source.DefiLlama, as: ExchangeRatesSourceDefiLlama + alias Explorer.Market.History.Source.Price.CryptoCompare + alias Explorer.Market.History.Source.TVL, as: SourceTVL + + @behaviour SourceTVL + + @impl SourceTVL + def fetch_tvl(previous_days) do + coin_id = Application.get_env(:explorer, Explorer.ExchangeRates.Source.DefiLlama, [])[:coin_id] + + if coin_id do + url = + ExchangeRatesSourceDefiLlama.history_url(previous_days) <> + "/" <> coin_id + + case Source.http_request(url, ExchangeRatesSourceDefiLlama.headers()) do + {:ok, data} -> + result = + data + |> format_data() + + {:ok, result} + + _ -> + :error + end + else + {:ok, []} + end + end + + @spec format_data(term()) :: SourceTVL.record() | nil + defp format_data(nil), do: nil + + defp format_data(data) do + Enum.map(data, fn %{"date" => date, "tvl" => tvl} -> + %{ + tvl: Decimal.new(to_string(tvl)), + date: CryptoCompare.date(date) + } + end) + end +end diff --git a/apps/explorer/lib/explorer/market/market_history.ex b/apps/explorer/lib/explorer/market/market_history.ex index aca3090625..0fabaf0e30 100644 --- a/apps/explorer/lib/explorer/market/market_history.ex +++ b/apps/explorer/lib/explorer/market/market_history.ex @@ -10,6 +10,7 @@ defmodule Explorer.Market.MarketHistory do field(:date, :date) field(:opening_price, :decimal) field(:market_cap, :decimal) + field(:tvl, :decimal) end @typedoc """ @@ -19,11 +20,13 @@ defmodule Explorer.Market.MarketHistory do * `:date` - The date in UTC. * `:opening_price` - Opening price in USD. * `:market_cap` - Market cap in USD. + * `:market_cap` - TVL in USD. """ @type t :: %__MODULE__{ closing_price: Decimal.t(), date: Date.t(), opening_price: Decimal.t(), - market_cap: Decimal.t() + market_cap: Decimal.t(), + tvl: Decimal.t() } end diff --git a/apps/explorer/priv/repo/migrations/20231003093553_add_tvl_to_market_history_table.exs b/apps/explorer/priv/repo/migrations/20231003093553_add_tvl_to_market_history_table.exs new file mode 100644 index 0000000000..5ff8dd795a --- /dev/null +++ b/apps/explorer/priv/repo/migrations/20231003093553_add_tvl_to_market_history_table.exs @@ -0,0 +1,9 @@ +defmodule Explorer.Repo.Migrations.AddTvlToMarketHistoryTable do + use Ecto.Migration + + def change do + alter table(:market_history) do + add(:tvl, :decimal) + end + end +end diff --git a/apps/explorer/test/explorer/market/history/cataloger_test.exs b/apps/explorer/test/explorer/market/history/cataloger_test.exs index 7ffe2de63f..f7a86a960e 100644 --- a/apps/explorer/test/explorer/market/history/cataloger_test.exs +++ b/apps/explorer/test/explorer/market/history/cataloger_test.exs @@ -63,45 +63,100 @@ defmodule Explorer.Market.History.CatalogerTest do assert_receive {_ref, {:price_history, {1, 0, {:ok, ^records}}}} end + test "handle_info with successful tasks (price, market cap and tvl)" do + Application.put_env(:explorer, Cataloger, history_fetch_interval: 1) + + price_records = [ + %{date: ~D[2018-04-01], closing_price: Decimal.new(10), opening_price: Decimal.new(5)}, + %{date: ~D[2018-04-02], closing_price: Decimal.new(6), opening_price: Decimal.new(2)} + ] + + market_cap_records = [%{date: ~D[2018-04-01], market_cap: Decimal.new(100_500)}] + tvl_records = [%{date: ~D[2018-04-01], tvl: Decimal.new(200_500)}] + + state = %{ + price_records: price_records + } + + state2 = Map.put(state, :market_cap_records, market_cap_records) + + state3 = Map.put(state2, :tvl_records, tvl_records) + + assert {:noreply, state} == Cataloger.handle_info({nil, {:price_history, {1, 0, {:ok, price_records}}}}, state) + assert_receive {:fetch_market_cap_history, 365} + + assert {:noreply, state2} == + Cataloger.handle_info({nil, {:market_cap_history, {0, 3, {:ok, market_cap_records}}}}, state) + + assert {:noreply, state3} == + Cataloger.handle_info({nil, {:tvl_history, {0, 3, {:ok, tvl_records}}}}, state2) + + assert record2 = Repo.get_by(MarketHistory, date: Enum.at(price_records, 1).date) + assert record1 = Repo.get_by(MarketHistory, date: Enum.at(price_records, 0).date) + assert record2.closing_price == Decimal.new(6) + assert record2.market_cap == nil + assert record2.tvl == nil + assert record1.closing_price == Decimal.new(10) + assert record1.market_cap == Decimal.new(100_500) + assert record1.tvl == Decimal.new(200_500) + end + test "handle_info with successful tasks (price and market cap)" do Application.put_env(:explorer, Cataloger, history_fetch_interval: 1) - record_price = %{date: ~D[2018-04-01], closing_price: Decimal.new(10), opening_price: Decimal.new(5)} - record_market_caps = [%{date: ~D[2018-04-01], market_cap: Decimal.new(100_500)}] + price_records = [%{date: ~D[2018-04-01], closing_price: Decimal.new(10), opening_price: Decimal.new(5)}] + market_cap_records = [%{date: ~D[2018-04-01], market_cap: Decimal.new(100_500)}] + tvl_records = [] state = %{ - price_records: [ - record_price - ] + price_records: price_records } - assert {:noreply, state} == Cataloger.handle_info({nil, {:price_history, {1, 0, {:ok, [record_price]}}}}, state) + state2 = Map.put(state, :market_cap_records, market_cap_records) + + state3 = Map.put(state2, :tvl_records, []) + + assert {:noreply, state} == Cataloger.handle_info({nil, {:price_history, {1, 0, {:ok, price_records}}}}, state) assert_receive {:fetch_market_cap_history, 365} - assert {:noreply, state} == - Cataloger.handle_info({nil, {:market_cap_history, {0, 3, {:ok, record_market_caps}}}}, state) + assert {:noreply, state2} == + Cataloger.handle_info({nil, {:market_cap_history, {0, 3, {:ok, market_cap_records}}}}, state) + + assert {:noreply, state3} == + Cataloger.handle_info({nil, {:tvl_history, {0, 3, {:ok, tvl_records}}}}, state2) - assert Repo.get_by(MarketHistory, date: record_price.date) + assert record = Repo.get_by(MarketHistory, date: Enum.at(price_records, 0).date) + assert record.opening_price == Decimal.new(5) + assert record.market_cap == Decimal.new(100_500) + assert record.tvl == nil end test "handle_info with successful price task" do Application.put_env(:explorer, Cataloger, history_fetch_interval: 1) - record_price = %{date: ~D[2018-04-01], closing_price: Decimal.new(10), opening_price: Decimal.new(5)} - record_market_cap = nil + price_records = [%{date: ~D[2018-04-01], closing_price: Decimal.new(10), opening_price: Decimal.new(5)}] + market_cap_records = [] + tvl_records = [] state = %{ - price_records: [ - record_price - ] + price_records: price_records } - assert {:noreply, state} == Cataloger.handle_info({nil, {:price_history, {1, 0, {:ok, [record_price]}}}}, state) + state2 = Map.put(state, :market_cap_records, market_cap_records) + + state3 = Map.put(state2, :tvl_records, tvl_records) + + assert {:noreply, state} == Cataloger.handle_info({nil, {:price_history, {1, 0, {:ok, price_records}}}}, state) assert_receive {:fetch_market_cap_history, 365} - assert {:noreply, state} == - Cataloger.handle_info({nil, {:market_cap_history, {0, 3, {:ok, record_market_cap}}}}, state) + assert {:noreply, state2} == + Cataloger.handle_info({nil, {:market_cap_history, {0, 3, {:ok, market_cap_records}}}}, state) + + assert {:noreply, state3} == + Cataloger.handle_info({nil, {:tvl_history, {0, 3, {:ok, tvl_records}}}}, state2) - assert record = Repo.get_by(MarketHistory, date: record_price.date) + assert record = Repo.get_by(MarketHistory, date: Enum.at(price_records, 0).date) + assert record.closing_price == Decimal.new(10) assert record.market_cap == nil + assert record.tvl == nil end test "handle info for DOWN message" do diff --git a/config/config_helper.exs b/config/config_helper.exs index 709b6455b2..9d4e91eb5f 100644 --- a/config/config_helper.exs +++ b/config/config_helper.exs @@ -1,7 +1,7 @@ defmodule ConfigHelper do import Bitwise alias Explorer.ExchangeRates.Source - alias Explorer.Market.History.Source.{MarketCap, Price} + alias Explorer.Market.History.Source.{MarketCap, Price, TVL} alias Indexer.Transform.Blocks def repos do @@ -113,6 +113,14 @@ defmodule ConfigHelper do end end + @spec exchange_rates_tvl_source() :: TVL.DefiLlama + def exchange_rates_tvl_source do + case System.get_env("EXCHANGE_RATES_TVL_SOURCE") do + "defillama" -> TVL.DefiLlama + _ -> TVL.DefiLlama + end + end + @spec exchange_rates_price_source() :: Price.CoinGecko | Price.CoinMarketCap | Price.CryptoCompare def exchange_rates_price_source do case System.get_env("EXCHANGE_RATES_PRICE_SOURCE") do diff --git a/config/runtime.exs b/config/runtime.exs index f709fce6ed..1edf64b850 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -280,7 +280,8 @@ config :explorer, Explorer.ExchangeRates, config :explorer, Explorer.ExchangeRates.Source, source: ConfigHelper.exchange_rates_source(), price_source: ConfigHelper.exchange_rates_price_source(), - market_cap_source: ConfigHelper.exchange_rates_market_cap_source() + market_cap_source: ConfigHelper.exchange_rates_market_cap_source(), + tvl_source: ConfigHelper.exchange_rates_tvl_source() config :explorer, Explorer.ExchangeRates.Source.CoinMarketCap, api_key: System.get_env("EXCHANGE_RATES_COINMARKETCAP_API_KEY"), @@ -291,6 +292,8 @@ config :explorer, Explorer.ExchangeRates.Source.CoinGecko, api_key: System.get_env("EXCHANGE_RATES_COINGECKO_API_KEY"), coin_id: System.get_env("EXCHANGE_RATES_COINGECKO_COIN_ID") +config :explorer, Explorer.ExchangeRates.Source.DefiLlama, coin_id: System.get_env("EXCHANGE_RATES_DEFILLAMA_COIN_ID") + config :explorer, Explorer.ExchangeRates.TokenExchangeRates, enabled: !ConfigHelper.parse_bool_env_var("DISABLE_TOKEN_EXCHANGE_RATE", "true"), interval: ConfigHelper.parse_time_env_var("TOKEN_EXCHANGE_RATE_INTERVAL", "5s"), diff --git a/cspell.json b/cspell.json index 0338069e35..94b5f97444 100644 --- a/cspell.json +++ b/cspell.json @@ -526,6 +526,7 @@ "lastmod", "qitmeer", "meer", + "DefiLlama" ], "enableFiletypes": [ "dotenv", diff --git a/docker-compose/envs/common-blockscout.env b/docker-compose/envs/common-blockscout.env index c20e65a313..f5edaabfea 100644 --- a/docker-compose/envs/common-blockscout.env +++ b/docker-compose/envs/common-blockscout.env @@ -34,6 +34,7 @@ EMISSION_FORMAT=DEFAULT COIN= EXCHANGE_RATES_COIN= # EXCHANGE_RATES_MARKET_CAP_SOURCE= +# EXCHANGE_RATES_TVL_SOURCE= # EXCHANGE_RATES_PRICE_SOURCE= # EXCHANGE_RATES_COINGECKO_COIN_ID= # EXCHANGE_RATES_COINGECKO_API_KEY=