DefiLlama TVL source

CHANGELOG.md

@@ -5,6 +5,7 @@
 ### Features
 
 - [#8472](https://github.com/blockscout/blockscout/pull/8472) - Integrate `/api/v2/bytecodes/sources:search-all` of `eth_bytecode_db`
+- [#8589](https://github.com/blockscout/blockscout/pull/8589) - DefiLlama TVL source
 - [#8544](https://github.com/blockscout/blockscout/pull/8544) - Fix `nil` `"structLogs"`
 - [#8561](https://github.com/blockscout/blockscout/pull/8561), [#8564](https://github.com/blockscout/blockscout/pull/8564) - Get historical market cap data from CoinGecko
 - [#8386](https://github.com/blockscout/blockscout/pull/8386) - Add `owner_address_hash` to the `token_instances`

apps/block_scout_web/lib/block_scout_web/controllers/api/v2/stats_controller.ex

@@ -112,7 +112,7 @@ defmodule BlockScoutWeb.API.V2.StatsController do
         data ->
           data
       end
-      |> Enum.map(fn day -> Map.take(day, [:closing_price, :market_cap, :date]) end)
+      |> Enum.map(fn day -> Map.take(day, [:closing_price, :market_cap, :tvl, :date]) end)
 
     market_history_data =
       MarketHistoryChartController.encode_market_history_data(price_history_data, current_total_supply)

apps/block_scout_web/lib/block_scout_web/controllers/chain/market_history_chart_controller.ex

@@ -59,7 +59,7 @@ defmodule BlockScoutWeb.Chain.MarketHistoryChartController do
 
       day
       |> Map.put(:market_cap, market_cap)
-      |> Map.take([:closing_price, :market_cap, :date])
+      |> Map.take([:closing_price, :market_cap, :tvl, :date])
     end)
     |> Jason.encode()
     |> case do

apps/explorer/lib/explorer/exchange_rates/source/defillama.ex

@@ -0,0 +1,46 @@
+defmodule Explorer.ExchangeRates.Source.DefiLlama do
+  @moduledoc """
+  Adapter for fetching exchange rates from https://defillama.com/
+
+  """
+
+  alias Explorer.ExchangeRates.Source
+
+  @behaviour Source
+
+  @impl Source
+  def format_data(_), do: []
+
+  @spec history_url(non_neg_integer()) :: String.t()
+  def history_url(_previous_days) do
+    "#{base_url()}/historicalChainTvl"
+  end
+
+  @impl Source
+  def source_url do
+    ""
+  end
+
+  @impl Source
+  def source_url(_) do
+    ""
+  end
+
+  @impl Source
+  def headers do
+    []
+  end
+
+  defp base_url do
+    base_free_url()
+  end
+
+  defp base_free_url do
+    config(:base_url) || "https://api.llama.fi/v2"
+  end
+
+  @spec config(atom()) :: term
+  defp config(key) do
+    Application.get_env(:explorer, __MODULE__, [])[key]
+  end
+end

apps/explorer/lib/explorer/market/history/cataloger.ex

@@ -19,6 +19,7 @@ defmodule Explorer.Market.History.Cataloger do
 
   @price_failed_attempts 10
   @market_cap_failed_attempts 3
+  @tvl_failed_attempts 3
 
   @impl GenServer
   def init(:ok) do
@@ -38,32 +39,61 @@ defmodule Explorer.Market.History.Cataloger do
     {:noreply, state}
   end
 
+  @impl GenServer
+  # Record fetch successful.
+  def handle_info({_ref, {:price_history, {_, _, {:ok, records}}}}, state) do
+    Process.send(self(), {:fetch_market_cap_history, 365}, [])
+    state = state |> Map.put_new(:price_records, records)
+
+    {:noreply, state}
+  end
+
   @impl GenServer
   def handle_info({:fetch_market_cap_history, day_count}, state) do
     fetch_market_cap_history(day_count)
+    state = state |> Map.put_new(:price_records, [])
 
     {:noreply, state}
   end
 
   @impl GenServer
   # Record fetch successful.
-  def handle_info({_ref, {:price_history, {_, _, {:ok, records}}}}, state) do
+  def handle_info({_ref, {:market_cap_history, {_, _, {:ok, nil}}}}, state) do
-    Process.send(self(), {:fetch_market_cap_history, 365}, [])
+    Process.send(self(), {:fetch_tvl_history, 365}, [])
-    state = state |> Map.put_new(:price_records, records)
+    state = state |> Map.put_new(:market_cap_records, [])
 
-    {:noreply, state |> Map.put_new(:price_records, state)}
+    {:noreply, state}
   end
 
   @impl GenServer
   # Record fetch successful.
-  def handle_info({_ref, {:market_cap_history, {_, _, {:ok, nil}}}}, state) do
+  def handle_info({_ref, {:market_cap_history, {_, _, {:ok, market_cap_records}}}}, state) do
-    market_cap_history(state.price_records, state)
+    Process.send(self(), {:fetch_tvl_history, 365}, [])
+    state = state |> Map.put_new(:market_cap_records, market_cap_records)
+
+    {:noreply, state}
+  end
+
+  @impl GenServer
+  def handle_info({:fetch_tvl_history, day_count}, state) do
+    fetch_tvl_history(day_count)
+
+    {:noreply, state}
   end
 
   @impl GenServer
   # Record fetch successful.
-  def handle_info({_ref, {:market_cap_history, {_, _, {:ok, market_cap_records}}}}, state) do
+  def handle_info({_ref, {:tvl_history, {_, _, {:ok, nil}}}}, state) do
-    records = compile_records(state.price_records, market_cap_records)
+    state = state |> Map.put_new(:tvl_records, [])
+    records = compile_records(state)
+    market_cap_history(records, state)
+  end
+
+  @impl GenServer
+  # Record fetch successful.
+  def handle_info({_ref, {:tvl_history, {_, _, {:ok, tvl_records}}}}, state) do
+    state = state |> Map.put_new(:tvl_records, tvl_records)
+    records = compile_records(state)
     market_cap_history(records, state)
   end
 
@@ -87,6 +117,16 @@ defmodule Explorer.Market.History.Cataloger do
     {:noreply, state}
   end
 
+  # Failed to get records. Try again.
+  @impl GenServer
+  def handle_info({_ref, {:tvl_history, {day_count, failed_attempts, :error}}}, state) do
+    Logger.warn(fn -> "Failed to fetch market cap history. Trying again." end)
+
+    fetch_tvl_history(day_count, failed_attempts + 1)
+
+    {:noreply, state}
+  end
+
   # Callback that a monitored process has shutdown.
   @impl GenServer
   def handle_info({:DOWN, _, :process, _, _}, state) do
@@ -135,6 +175,15 @@ defmodule Explorer.Market.History.Cataloger do
     )
   end
 
+  @spec source_tvl() :: module()
+  defp source_tvl do
+    config_or_default(
+      :tvl_source,
+      Explorer.ExchangeRates.Source,
+      Explorer.Market.History.Source.TVL.DefiLlama
+    )
+  end
+
   @spec fetch_price_history(non_neg_integer(), non_neg_integer()) :: Task.t()
   defp fetch_price_history(day_count, failed_attempts \\ 0) do
     Task.Supervisor.async_nolink(Explorer.MarketTaskSupervisor, fn ->
@@ -161,11 +210,31 @@ defmodule Explorer.Market.History.Cataloger do
     end)
   end
 
-  defp compile_records(price_records, market_cap_records) do
+  @spec fetch_tvl_history(non_neg_integer()) :: Task.t()
-    price_records
+  defp fetch_tvl_history(day_count, failed_attempts \\ 0) do
-    |> Enum.zip(market_cap_records)
+    Task.Supervisor.async_nolink(Explorer.MarketTaskSupervisor, fn ->
-    |> Enum.map(fn {price_map, market_cap_map} ->
+      Process.sleep(HistoryProcess.delay(failed_attempts))
-      Map.merge(price_map, market_cap_map)
+
+      if failed_attempts < @tvl_failed_attempts do
+        {:tvl_history, {day_count, failed_attempts, source_tvl().fetch_tvl(day_count)}}
+      else
+        {:tvl_history, {day_count, failed_attempts, {:ok, nil}}}
+      end
+    end)
+  end
+
+  defp compile_records(state) do
+    price_records = state.price_records
+    market_cap_records = state.market_cap_records
+    tvl_records = state.tvl_records
+
+    all_records = price_records ++ market_cap_records ++ tvl_records
+
+    all_records
+    |> Enum.group_by(fn %{date: date} -> date end)
+    |> Map.values()
+    |> Enum.map(fn a ->
+      Enum.reduce(a, %{}, fn x, acc -> Map.merge(x, acc) end)
     end)
   end
 end

apps/explorer/lib/explorer/market/history/source/tvl.ex

@@ -0,0 +1,18 @@
+defmodule Explorer.Market.History.Source.TVL do
+  @moduledoc """
+  Interface for a source that allows for fetching of TVL history.
+  """
+
+  @typedoc """
+  Record of market values for a specific date.
+  """
+  @type record :: %{
+          date: Date.t(),
+          tvl: Decimal.t()
+        }
+
+  @doc """
+  Fetch history for a specified amount of days in the past.
+  """
+  @callback fetch_tvl(previous_days :: non_neg_integer()) :: {:ok, [record()]} | :error
+end

apps/explorer/lib/explorer/market/history/source/tvl/defillama.ex

@@ -0,0 +1,49 @@
+defmodule Explorer.Market.History.Source.TVL.DefiLlama do
+  @moduledoc """
+  Adapter for fetching current market from DefiLlama.
+  """
+
+  alias Explorer.ExchangeRates.Source
+  alias Explorer.ExchangeRates.Source.DefiLlama, as: ExchangeRatesSourceDefiLlama
+  alias Explorer.Market.History.Source.Price.CryptoCompare
+  alias Explorer.Market.History.Source.TVL, as: SourceTVL
+
+  @behaviour SourceTVL
+
+  @impl SourceTVL
+  def fetch_tvl(previous_days) do
+    coin_id = Application.get_env(:explorer, Explorer.ExchangeRates.Source.DefiLlama, [])[:coin_id]
+
+    if coin_id do
+      url =
+        ExchangeRatesSourceDefiLlama.history_url(previous_days) <>
+          "/" <> coin_id
+
+      case Source.http_request(url, ExchangeRatesSourceDefiLlama.headers()) do
+        {:ok, data} ->
+          result =
+            data
+            |> format_data()
+
+          {:ok, result}
+
+        _ ->
+          :error
+      end
+    else
+      {:ok, []}
+    end
+  end
+
+  @spec format_data(term()) :: SourceTVL.record() | nil
+  defp format_data(nil), do: nil
+
+  defp format_data(data) do
+    Enum.map(data, fn %{"date" => date, "tvl" => tvl} ->
+      %{
+        tvl: Decimal.new(to_string(tvl)),
+        date: CryptoCompare.date(date)
+      }
+    end)
+  end
+end

apps/explorer/lib/explorer/market/market_history.ex

@@ -10,6 +10,7 @@ defmodule Explorer.Market.MarketHistory do
     field(:date, :date)
     field(:opening_price, :decimal)
     field(:market_cap, :decimal)
+    field(:tvl, :decimal)
   end
 
   @typedoc """
@@ -19,11 +20,13 @@ defmodule Explorer.Market.MarketHistory do
    * `:date` - The date in UTC.
    * `:opening_price` - Opening price in USD.
    * `:market_cap` - Market cap in USD.
+   * `:market_cap` - TVL in USD.
   """
   @type t :: %__MODULE__{
           closing_price: Decimal.t(),
           date: Date.t(),
           opening_price: Decimal.t(),
-          market_cap: Decimal.t()
+          market_cap: Decimal.t(),
+          tvl: Decimal.t()
         }
 end

apps/explorer/priv/repo/migrations/20231003093553_add_tvl_to_market_history_table.exs

@@ -0,0 +1,9 @@
+defmodule Explorer.Repo.Migrations.AddTvlToMarketHistoryTable do
+  use Ecto.Migration
+
+  def change do
+    alter table(:market_history) do
+      add(:tvl, :decimal)
+    end
+  end
+end

apps/explorer/test/explorer/market/history/cataloger_test.exs

@@ -63,45 +63,100 @@ defmodule Explorer.Market.History.CatalogerTest do
     assert_receive {_ref, {:price_history, {1, 0, {:ok, ^records}}}}
   end
 
+  test "handle_info with successful tasks (price, market cap and tvl)" do
+    Application.put_env(:explorer, Cataloger, history_fetch_interval: 1)
+
+    price_records = [
+      %{date: ~D[2018-04-01], closing_price: Decimal.new(10), opening_price: Decimal.new(5)},
+      %{date: ~D[2018-04-02], closing_price: Decimal.new(6), opening_price: Decimal.new(2)}
+    ]
+
+    market_cap_records = [%{date: ~D[2018-04-01], market_cap: Decimal.new(100_500)}]
+    tvl_records = [%{date: ~D[2018-04-01], tvl: Decimal.new(200_500)}]
+
+    state = %{
+      price_records: price_records
+    }
+
+    state2 = Map.put(state, :market_cap_records, market_cap_records)
+
+    state3 = Map.put(state2, :tvl_records, tvl_records)
+
+    assert {:noreply, state} == Cataloger.handle_info({nil, {:price_history, {1, 0, {:ok, price_records}}}}, state)
+    assert_receive {:fetch_market_cap_history, 365}
+
+    assert {:noreply, state2} ==
+             Cataloger.handle_info({nil, {:market_cap_history, {0, 3, {:ok, market_cap_records}}}}, state)
+
+    assert {:noreply, state3} ==
+             Cataloger.handle_info({nil, {:tvl_history, {0, 3, {:ok, tvl_records}}}}, state2)
+
+    assert record2 = Repo.get_by(MarketHistory, date: Enum.at(price_records, 1).date)
+    assert record1 = Repo.get_by(MarketHistory, date: Enum.at(price_records, 0).date)
+    assert record2.closing_price == Decimal.new(6)
+    assert record2.market_cap == nil
+    assert record2.tvl == nil
+    assert record1.closing_price == Decimal.new(10)
+    assert record1.market_cap == Decimal.new(100_500)
+    assert record1.tvl == Decimal.new(200_500)
+  end
+
   test "handle_info with successful tasks (price and market cap)" do
     Application.put_env(:explorer, Cataloger, history_fetch_interval: 1)
-    record_price = %{date: ~D[2018-04-01], closing_price: Decimal.new(10), opening_price: Decimal.new(5)}
+    price_records = [%{date: ~D[2018-04-01], closing_price: Decimal.new(10), opening_price: Decimal.new(5)}]
-    record_market_caps = [%{date: ~D[2018-04-01], market_cap: Decimal.new(100_500)}]
+    market_cap_records = [%{date: ~D[2018-04-01], market_cap: Decimal.new(100_500)}]
+    tvl_records = []
 
     state = %{
-      price_records: [
+      price_records: price_records
-        record_price
-      ]
     }
 
-    assert {:noreply, state} == Cataloger.handle_info({nil, {:price_history, {1, 0, {:ok, [record_price]}}}}, state)
+    state2 = Map.put(state, :market_cap_records, market_cap_records)
+
+    state3 = Map.put(state2, :tvl_records, [])
+
+    assert {:noreply, state} == Cataloger.handle_info({nil, {:price_history, {1, 0, {:ok, price_records}}}}, state)
     assert_receive {:fetch_market_cap_history, 365}
 
-    assert {:noreply, state} ==
+    assert {:noreply, state2} ==
-             Cataloger.handle_info({nil, {:market_cap_history, {0, 3, {:ok, record_market_caps}}}}, state)
+             Cataloger.handle_info({nil, {:market_cap_history, {0, 3, {:ok, market_cap_records}}}}, state)
+
+    assert {:noreply, state3} ==
+             Cataloger.handle_info({nil, {:tvl_history, {0, 3, {:ok, tvl_records}}}}, state2)
 
-    assert Repo.get_by(MarketHistory, date: record_price.date)
+    assert record = Repo.get_by(MarketHistory, date: Enum.at(price_records, 0).date)
+    assert record.opening_price == Decimal.new(5)
+    assert record.market_cap == Decimal.new(100_500)
+    assert record.tvl == nil
   end
 
   test "handle_info with successful price task" do
     Application.put_env(:explorer, Cataloger, history_fetch_interval: 1)
-    record_price = %{date: ~D[2018-04-01], closing_price: Decimal.new(10), opening_price: Decimal.new(5)}
+    price_records = [%{date: ~D[2018-04-01], closing_price: Decimal.new(10), opening_price: Decimal.new(5)}]
-    record_market_cap = nil
+    market_cap_records = []
+    tvl_records = []
 
     state = %{
-      price_records: [
+      price_records: price_records
-        record_price
-      ]
     }
 
-    assert {:noreply, state} == Cataloger.handle_info({nil, {:price_history, {1, 0, {:ok, [record_price]}}}}, state)
+    state2 = Map.put(state, :market_cap_records, market_cap_records)
+
+    state3 = Map.put(state2, :tvl_records, tvl_records)
+
+    assert {:noreply, state} == Cataloger.handle_info({nil, {:price_history, {1, 0, {:ok, price_records}}}}, state)
     assert_receive {:fetch_market_cap_history, 365}
 
-    assert {:noreply, state} ==
+    assert {:noreply, state2} ==
-             Cataloger.handle_info({nil, {:market_cap_history, {0, 3, {:ok, record_market_cap}}}}, state)
+             Cataloger.handle_info({nil, {:market_cap_history, {0, 3, {:ok, market_cap_records}}}}, state)
+
+    assert {:noreply, state3} ==
+             Cataloger.handle_info({nil, {:tvl_history, {0, 3, {:ok, tvl_records}}}}, state2)
 
-    assert record = Repo.get_by(MarketHistory, date: record_price.date)
+    assert record = Repo.get_by(MarketHistory, date: Enum.at(price_records, 0).date)
+    assert record.closing_price == Decimal.new(10)
     assert record.market_cap == nil
+    assert record.tvl == nil
   end
 
   test "handle info for DOWN message" do

config/config_helper.exs

@@ -1,7 +1,7 @@
 defmodule ConfigHelper do
   import Bitwise
   alias Explorer.ExchangeRates.Source
-  alias Explorer.Market.History.Source.{MarketCap, Price}
+  alias Explorer.Market.History.Source.{MarketCap, Price, TVL}
   alias Indexer.Transform.Blocks
 
   def repos do
@@ -113,6 +113,14 @@ defmodule ConfigHelper do
     end
   end
 
+  @spec exchange_rates_tvl_source() :: TVL.DefiLlama
+  def exchange_rates_tvl_source do
+    case System.get_env("EXCHANGE_RATES_TVL_SOURCE") do
+      "defillama" -> TVL.DefiLlama
+      _ -> TVL.DefiLlama
+    end
+  end
+
   @spec exchange_rates_price_source() :: Price.CoinGecko | Price.CoinMarketCap | Price.CryptoCompare
   def exchange_rates_price_source do
     case System.get_env("EXCHANGE_RATES_PRICE_SOURCE") do

config/runtime.exs

@@ -280,7 +280,8 @@ config :explorer, Explorer.ExchangeRates,
 config :explorer, Explorer.ExchangeRates.Source,
   source: ConfigHelper.exchange_rates_source(),
   price_source: ConfigHelper.exchange_rates_price_source(),
-  market_cap_source: ConfigHelper.exchange_rates_market_cap_source()
+  market_cap_source: ConfigHelper.exchange_rates_market_cap_source(),
+  tvl_source: ConfigHelper.exchange_rates_tvl_source()
 
 config :explorer, Explorer.ExchangeRates.Source.CoinMarketCap,
   api_key: System.get_env("EXCHANGE_RATES_COINMARKETCAP_API_KEY"),
@@ -291,6 +292,8 @@ config :explorer, Explorer.ExchangeRates.Source.CoinGecko,
   api_key: System.get_env("EXCHANGE_RATES_COINGECKO_API_KEY"),
   coin_id: System.get_env("EXCHANGE_RATES_COINGECKO_COIN_ID")
 
+config :explorer, Explorer.ExchangeRates.Source.DefiLlama, coin_id: System.get_env("EXCHANGE_RATES_DEFILLAMA_COIN_ID")
+
 config :explorer, Explorer.ExchangeRates.TokenExchangeRates,
   enabled: !ConfigHelper.parse_bool_env_var("DISABLE_TOKEN_EXCHANGE_RATE", "true"),
   interval: ConfigHelper.parse_time_env_var("TOKEN_EXCHANGE_RATE_INTERVAL", "5s"),

cspell.json

@@ -526,6 +526,7 @@
         "lastmod",
         "qitmeer",
         "meer",
+        "DefiLlama"
     ],
     "enableFiletypes": [
         "dotenv",

docker-compose/envs/common-blockscout.env

@@ -34,6 +34,7 @@ EMISSION_FORMAT=DEFAULT
 COIN=
 EXCHANGE_RATES_COIN=
 # EXCHANGE_RATES_MARKET_CAP_SOURCE=
+# EXCHANGE_RATES_TVL_SOURCE=
 # EXCHANGE_RATES_PRICE_SOURCE=
 # EXCHANGE_RATES_COINGECKO_COIN_ID=
 # EXCHANGE_RATES_COINGECKO_API_KEY=