From e12b010a0eab8834dfda05d0713014bf6b20c3f7 Mon Sep 17 00:00:00 2001 From: Fedor Ivanov Date: Tue, 10 Sep 2024 14:35:41 +0300 Subject: [PATCH] feat: support for filecoin native addresses (#10468) * feat: implement `Ecto` type for filecoin address * fix: use proper hashing algorithm for checksum * refactor: avoid hardcoding * feat: add `NativeAddress.ID` type * chore: add `alias Blake2.Blake2b` to fix credo * feat: implement `Ecto` type for filecoin address * chore: rename id address module * feat: fix formatting * feat: add a table for pending address operations * feat: add filecoin fields to addresses relation * feat: create pending operation when new address is imported * feat: implement filecoin native address fetcher * chore: remove merge artifacts * fix: cspell * fix: alias in `native_address_test.exs` * fix: cspell * fix: lock address and corresponding operation for update * feat: trigger async fetch of address info from block fetcher * fix: compilation deadlock * fix: add fetcher supervisor case * feat: add migrator * fix: create pending address operation even if the address exists * feat: render filecoin address info in API v2 views * fix: user controller test * feat: add gauge metric for pending address operations * feat: save http error code for failed fetches * chore: rename fetcher * fix: rebase artifacts * chore: list migrator envs in `common-blockscout.env` * chore: process review comments by @vbaranov * chore: migrate from `blake2_elixir` to `blake2` package * chore: reduce log level to `debug` * chore: set infinity timeout for gauge metric query * refactor: remove redundant `Multi` in filling migration --- .../views/api/v2/filecoin_view.ex | 35 ++ .../block_scout_web/views/api/v2/helper.ex | 14 + .../account/api/v2/user_controller_test.exs | 20 +- apps/explorer/lib/explorer/application.ex | 5 +- apps/explorer/lib/explorer/chain/address.ex | 186 +++++--- .../lib/explorer/chain/filecoin/id.ex | 157 +++++++ .../explorer/chain/filecoin/native_address.ex | 408 ++++++++++++++++++ .../filecoin/pending_address_operation.ex | 73 ++++ apps/explorer/lib/explorer/chain/hash.ex | 2 +- .../explorer/chain/import/runner/addresses.ex | 42 +- .../explorer/chain/import/runner/blocks.ex | 9 +- .../explorer/chain/import/runner/helper.ex | 22 + .../filecoin_pending_address_operations.ex | 71 +++ apps/explorer/mix.exs | 4 +- ...4142_create_pending_address_operations.exs | 23 + ...34138_add_chain_type_fields_to_address.exs | 11 + .../chain/filecoin/native_address_test.exs | 175 ++++++++ apps/indexer/lib/indexer/application.ex | 22 +- .../lib/indexer/block/catchup/fetcher.ex | 2 + apps/indexer/lib/indexer/block/fetcher.ex | 12 +- .../lib/indexer/block/realtime/fetcher.ex | 2 + .../indexer/fetcher/filecoin/address_info.ex | 215 +++++++++ .../lib/indexer/fetcher/filecoin/beryx_api.ex | 50 +++ ...in_pending_address_operations_collector.ex | 29 ++ .../pending_block_operations_collector.ex | 2 +- apps/indexer/lib/indexer/supervisor.ex | 3 + .../indexer/block/catchup/fetcher_test.exs | 1 + .../test/indexer/block/fetcher_test.exs | 4 + .../indexer/block/realtime/fetcher_test.exs | 4 + ...filecoin_native_address_supervisor_case.ex | 17 + config/runtime.exs | 23 + cspell.json | 5 + docker-compose/envs/common-blockscout.env | 7 + mix.lock | 1 + 34 files changed, 1579 insertions(+), 77 deletions(-) create mode 100644 apps/block_scout_web/lib/block_scout_web/views/api/v2/filecoin_view.ex create mode 100644 apps/explorer/lib/explorer/chain/filecoin/id.ex create mode 100644 apps/explorer/lib/explorer/chain/filecoin/native_address.ex create mode 100644 apps/explorer/lib/explorer/chain/filecoin/pending_address_operation.ex create mode 100644 apps/explorer/lib/explorer/chain/import/runner/helper.ex create mode 100644 apps/explorer/lib/explorer/migrator/filecoin_pending_address_operations.ex create mode 100644 apps/explorer/priv/filecoin/migrations/20240801134142_create_pending_address_operations.exs create mode 100644 apps/explorer/priv/filecoin/migrations/20240807134138_add_chain_type_fields_to_address.exs create mode 100644 apps/explorer/test/explorer/chain/filecoin/native_address_test.exs create mode 100644 apps/indexer/lib/indexer/fetcher/filecoin/address_info.ex create mode 100644 apps/indexer/lib/indexer/fetcher/filecoin/beryx_api.ex create mode 100644 apps/indexer/lib/indexer/prometheus/collector/filecoin_pending_address_operations_collector.ex rename apps/indexer/lib/indexer/prometheus/{ => collector}/pending_block_operations_collector.ex (91%) create mode 100644 apps/indexer/test/support/indexer/fetcher/filecoin_native_address_supervisor_case.ex diff --git a/apps/block_scout_web/lib/block_scout_web/views/api/v2/filecoin_view.ex b/apps/block_scout_web/lib/block_scout_web/views/api/v2/filecoin_view.ex new file mode 100644 index 0000000000..191fbf6f69 --- /dev/null +++ b/apps/block_scout_web/lib/block_scout_web/views/api/v2/filecoin_view.ex @@ -0,0 +1,35 @@ +defmodule BlockScoutWeb.API.V2.FilecoinView do + @moduledoc """ + View functions for rendering Filecoin-related data in JSON format. + """ + + alias Explorer.Chain.Address + + @doc """ + Extends the json output with a sub-map containing information related to + Filecoin native addressing. + """ + @spec extend_address_json_response(map(), Address.t()) :: map() + def extend_address_json_response(result, %Address{} = address) do + filecoin_id = Map.get(address, :filecoin_id) + filecoin_robust = Map.get(address, :filecoin_robust) + filecoin_actor_type = Map.get(address, :filecoin_actor_type) + + is_fetched = + Enum.all?( + [ + filecoin_id, + filecoin_robust, + filecoin_actor_type + ], + &(not is_nil(&1)) + ) + + Map.put(result, :filecoin, %{ + is_fetched: is_fetched, + id: filecoin_id, + robust: filecoin_robust, + actor_type: filecoin_actor_type + }) + end +end diff --git a/apps/block_scout_web/lib/block_scout_web/views/api/v2/helper.ex b/apps/block_scout_web/lib/block_scout_web/views/api/v2/helper.ex index 5f483b1911..8df0acb81b 100644 --- a/apps/block_scout_web/lib/block_scout_web/views/api/v2/helper.ex +++ b/apps/block_scout_web/lib/block_scout_web/views/api/v2/helper.ex @@ -90,6 +90,7 @@ defmodule BlockScoutWeb.API.V2.Helper do "ens_domain_name" => address.ens_domain_name, "metadata" => address.metadata } + |> address_chain_type_fields(address) end def address_with_info(%NotLoaded{}, address_hash) do @@ -120,6 +121,19 @@ defmodule BlockScoutWeb.API.V2.Helper do } end + case Application.compile_env(:explorer, :chain_type) do + :filecoin -> + defp address_chain_type_fields(result, address) do + # credo:disable-for-next-line Credo.Check.Design.AliasUsage + BlockScoutWeb.API.V2.FilecoinView.extend_address_json_response(result, address) + end + + _ -> + defp address_chain_type_fields(result, _address) do + result + end + end + defp minimal_proxy_pattern?(proxy_implementations) do proxy_implementations.proxy_type == :eip1167 end diff --git a/apps/block_scout_web/test/block_scout_web/controllers/account/api/v2/user_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/account/api/v2/user_controller_test.exs index 30f44f611b..cfeeaa3101 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/account/api/v2/user_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/account/api/v2/user_controller_test.exs @@ -182,7 +182,9 @@ defmodule BlockScoutWeb.Account.Api.V2.UserControllerTest do |> Map.get("items") assert Enum.all?(created, fn {_, _, map} -> - map in response + Enum.any?(response, fn item -> + addresses_json_match?(map, item) + end) end) end @@ -237,7 +239,11 @@ defmodule BlockScoutWeb.Account.Api.V2.UserControllerTest do |> json_response(200) |> Map.get("items") - assert Enum.all?(created, fn {_, _, map} -> map in response end) + assert Enum.all?(created, fn {_, _, map} -> + Enum.any?(response, fn item -> + addresses_json_match?(map, item) + end) + end) {_, _, %{"id" => id}} = Enum.at(created, 0) @@ -1264,4 +1270,14 @@ defmodule BlockScoutWeb.Account.Api.V2.UserControllerTest do assert second_page_resp["next_page_params"] == nil compare_item(Enum.at(list, 0), Enum.at(second_page_resp["items"], 0)) end + + defp addresses_json_match?(expected, actual) do + Enum.all?(expected, fn {key, value} -> + case value do + # Recursively compare nested maps + %{} -> addresses_json_match?(value, actual[key]) + _ -> actual[key] == value + end + end) + end end diff --git a/apps/explorer/lib/explorer/application.ex b/apps/explorer/lib/explorer/application.ex index 8159ef8179..ba57a31405 100644 --- a/apps/explorer/lib/explorer/application.ex +++ b/apps/explorer/lib/explorer/application.ex @@ -142,8 +142,9 @@ defmodule Explorer.Application do configure(Explorer.Migrator.TransactionBlockConsensus), configure(Explorer.Migrator.TokenTransferBlockConsensus), configure(Explorer.Migrator.RestoreOmittedWETHTransfers), - configure_chain_type_dependent_process(Explorer.Chain.Cache.StabilityValidatorsCounters, :stability), - configure_mode_dependent_process(Explorer.Migrator.ShrinkInternalTransactions, :indexer) + configure(Explorer.Migrator.FilecoinPendingAddressOperations), + configure_mode_dependent_process(Explorer.Migrator.ShrinkInternalTransactions, :indexer), + configure_chain_type_dependent_process(Explorer.Chain.Cache.StabilityValidatorsCounters, :stability) ] |> List.flatten() diff --git a/apps/explorer/lib/explorer/chain/address.ex b/apps/explorer/lib/explorer/chain/address.ex index e931970698..52c1d662a9 100644 --- a/apps/explorer/lib/explorer/chain/address.ex +++ b/apps/explorer/lib/explorer/chain/address.ex @@ -1,15 +1,10 @@ -defmodule Explorer.Chain.Address do +defmodule Explorer.Chain.Address.Schema do @moduledoc """ - A stored representation of a web3 address. - """ - - require Bitwise + A stored representation of a web3 address. - use Explorer.Schema - - alias Ecto.Association.NotLoaded - alias Ecto.Changeset - alias Explorer.{Chain, PagingOptions, Repo} + Changes in the schema should be reflected in the bulk import module: + - Explorer.Chain.Import.Runner.Addresses + """ alias Explorer.Chain.{ Address, @@ -25,13 +20,128 @@ defmodule Explorer.Chain.Address do Withdrawal } + alias Explorer.Chain.Cache.{Accounts, NetVersion} + alias Explorer.Chain.SmartContract.Proxy.Models.Implementation + + @chain_type_fields (case Application.compile_env(:explorer, :chain_type) do + :filecoin -> + alias Explorer.Chain.Filecoin.{IDAddress, NativeAddress} + + quote do + [ + field(:filecoin_id, IDAddress), + field(:filecoin_robust, NativeAddress), + field( + :filecoin_actor_type, + Ecto.Enum, + values: + Enum.with_index([ + :account, + :cron, + :datacap, + :eam, + :ethaccount, + :evm, + :init, + :market, + :miner, + :multisig, + :paych, + :placeholder, + :power, + :reward, + :system, + :verifreg + ]) + ) + ] + end + + _ -> + [] + end) + + defmacro generate do + quote do + @primary_key false + @primary_key false + typed_schema "addresses" do + field(:hash, Hash.Address, primary_key: true) + field(:fetched_coin_balance, Wei) + field(:fetched_coin_balance_block_number, :integer) :: Block.block_number() | nil + field(:contract_code, Data) + field(:nonce, :integer) + field(:decompiled, :boolean, default: false) + field(:verified, :boolean, default: false) + field(:has_decompiled_code?, :boolean, virtual: true) + field(:stale?, :boolean, virtual: true) + field(:transactions_count, :integer) + field(:token_transfers_count, :integer) + field(:gas_used, :integer) + field(:ens_domain_name, :string, virtual: true) + field(:metadata, :any, virtual: true) + + # todo: remove virtual field for a single implementation when frontend is bound to "implementations" object value in API + field(:implementation, :any, virtual: true) + + has_one(:smart_contract, SmartContract, references: :hash) + has_one(:token, Token, foreign_key: :contract_address_hash, references: :hash) + has_one(:proxy_implementations, Implementation, foreign_key: :proxy_address_hash, references: :hash) + + has_one( + :contracts_creation_internal_transaction, + InternalTransaction, + foreign_key: :created_contract_address_hash, + references: :hash + ) + + has_one( + :contracts_creation_transaction, + Transaction, + foreign_key: :created_contract_address_hash, + references: :hash + ) + + has_many(:names, Address.Name, foreign_key: :address_hash, references: :hash) + has_many(:decompiled_smart_contracts, DecompiledSmartContract, foreign_key: :address_hash, references: :hash) + has_many(:withdrawals, Withdrawal, foreign_key: :address_hash, references: :hash) + + timestamps() + + unquote_splicing(@chain_type_fields) + end + end + end +end + +defmodule Explorer.Chain.Address do + @moduledoc """ + A stored representation of a web3 address. + """ + + require Bitwise + require Explorer.Chain.Address.Schema + + use Explorer.Schema + + alias Ecto.Association.NotLoaded + alias Ecto.Changeset alias Explorer.Chain.Cache.{Accounts, NetVersion} alias Explorer.Chain.SmartContract.Proxy alias Explorer.Chain.SmartContract.Proxy.Models.Implementation + alias Explorer.Chain.{Address, Hash} + alias Explorer.{Chain, PagingOptions, Repo} @optional_attrs ~w(contract_code fetched_coin_balance fetched_coin_balance_block_number nonce decompiled verified gas_used transactions_count token_transfers_count)a + @chain_type_optional_attrs (case Application.compile_env(:explorer, :chain_type) do + :filecoin -> + ~w(filecoin_id filecoin_robust filecoin_actor_type)a + + _ -> + [] + end) @required_attrs ~w(hash)a - @allowed_attrs @optional_attrs ++ @required_attrs + @allowed_attrs @optional_attrs ++ @required_attrs ++ @chain_type_optional_attrs @typedoc """ Hash of the public key for this address. @@ -74,54 +184,18 @@ defmodule Explorer.Chain.Address do * `inserted_at` - when this address was inserted * `updated_at` - when this address was last updated * `ens_domain_name` - virtual field for ENS domain name passing - + #{case Application.compile_env(:explorer, :chain_type) do + :filecoin -> """ + * `filecoin_native_address` - robust f0/f1/f2/f3/f4 Filecoin address + * `filecoin_id_address` - short f0 Filecoin address that may change during chain reorgs + * `filecoin_actor_type` - type of actor associated with the Filecoin address + """ + _ -> "" + end} `fetched_coin_balance` and `fetched_coin_balance_block_number` may be updated when a new coin_balance row is fetched. They may also be updated when the balance is fetched via the on demand fetcher. """ - @primary_key false - typed_schema "addresses" do - field(:hash, Hash.Address, primary_key: true) - field(:fetched_coin_balance, Wei) - field(:fetched_coin_balance_block_number, :integer) :: Block.block_number() | nil - field(:contract_code, Data) - field(:nonce, :integer) - field(:decompiled, :boolean, default: false) - field(:verified, :boolean, default: false) - field(:has_decompiled_code?, :boolean, virtual: true) - field(:stale?, :boolean, virtual: true) - field(:transactions_count, :integer) - field(:token_transfers_count, :integer) - field(:gas_used, :integer) - field(:ens_domain_name, :string, virtual: true) - field(:metadata, :any, virtual: true) - - # todo: remove virtual field for a single implementation when frontend is bound to "implementations" object value in API - field(:implementation, :any, virtual: true) - - has_one(:smart_contract, SmartContract, references: :hash) - has_one(:token, Token, foreign_key: :contract_address_hash, references: :hash) - has_one(:proxy_implementations, Implementation, foreign_key: :proxy_address_hash, references: :hash) - - has_one( - :contracts_creation_internal_transaction, - InternalTransaction, - foreign_key: :created_contract_address_hash, - references: :hash - ) - - has_one( - :contracts_creation_transaction, - Transaction, - foreign_key: :created_contract_address_hash, - references: :hash - ) - - has_many(:names, Address.Name, foreign_key: :address_hash, references: :hash) - has_many(:decompiled_smart_contracts, DecompiledSmartContract, foreign_key: :address_hash, references: :hash) - has_many(:withdrawals, Withdrawal, foreign_key: :address_hash, references: :hash) - - timestamps() - end + Explorer.Chain.Address.Schema.generate() @balance_changeset_required_attrs @required_attrs ++ ~w(fetched_coin_balance fetched_coin_balance_block_number)a diff --git a/apps/explorer/lib/explorer/chain/filecoin/id.ex b/apps/explorer/lib/explorer/chain/filecoin/id.ex new file mode 100644 index 0000000000..74ce571e75 --- /dev/null +++ b/apps/explorer/lib/explorer/chain/filecoin/id.ex @@ -0,0 +1,157 @@ +defmodule Explorer.Chain.Filecoin.IDAddress do + @moduledoc """ + Handles Filecoin ID addresses, wrapping the `NativeAddress` type. + """ + + alias Explorer.Chain.Filecoin.NativeAddress + alias Poison.Encoder.BitString + + require Integer + + defstruct ~w(value)a + + @protocol_indicator 0 + + use Ecto.Type + + @type t :: %__MODULE__{value: binary()} + + @impl Ecto.Type + @spec type() :: :binary + def type, do: :binary + + defp to_native_address(%__MODULE__{value: value}) do + %NativeAddress{ + protocol_indicator: @protocol_indicator, + payload: value + } + end + + @doc """ + Casts a binary string to a `Explorer.Chain.Filecoin.IDAddress`. + + ## Examples + + iex> Explorer.Chain.Filecoin.IDAddress.cast("f01729") + {:ok, %Explorer.Chain.Filecoin.IDAddress{value: <<193, 13>>}} + + iex> Explorer.Chain.Filecoin.IDAddress.cast(%Explorer.Chain.Filecoin.IDAddress{value: <<193, 13>>}) + {:ok, %Explorer.Chain.Filecoin.IDAddress{value: <<193, 13>>}} + + iex> Explorer.Chain.Filecoin.IDAddress.cast("invalid") + :error + """ + @impl Ecto.Type + def cast(address_string) when is_binary(address_string) do + address_string + |> NativeAddress.cast() + |> case do + {:ok, + %NativeAddress{ + protocol_indicator: @protocol_indicator, + payload: value + }} -> + {:ok, %__MODULE__{value: value}} + + :error -> + :error + end + end + + @impl Ecto.Type + def cast(%__MODULE__{} = address), do: {:ok, address} + + @impl Ecto.Type + def cast(_), do: :error + + @doc """ + Dumps an `Explorer.Chain.Filecoin.IDAddress` to its binary representation. + + ## Examples + + iex> address = %Explorer.Chain.Filecoin.IDAddress{value: <<193, 13>>} + iex> Explorer.Chain.Filecoin.IDAddress.dump(address) + {:ok, <<0, 193, 13>>} + + iex> Explorer.Chain.Filecoin.IDAddress.dump("invalid") + :error + """ + @impl Ecto.Type + def dump(%__MODULE__{} = address) do + address + |> to_native_address() + |> NativeAddress.dump() + end + + def dump(_), do: :error + + @doc """ + Loads a binary representation of an `Explorer.Chain.Filecoin.IDAddress`. + + ## Examples + + iex> Explorer.Chain.Filecoin.IDAddress.load(<<0, 193, 13>>) + {:ok, %Explorer.Chain.Filecoin.IDAddress{value: <<193, 13>>}} + + iex> Explorer.Chain.Filecoin.IDAddress.load("invalid") + :error + """ + @impl Ecto.Type + def load(bytes) when is_binary(bytes) do + bytes + |> NativeAddress.load() + |> case do + {:ok, + %NativeAddress{ + protocol_indicator: @protocol_indicator, + payload: value + }} -> + {:ok, %__MODULE__{value: value}} + + _ -> + :error + end + end + + def load(_), do: :error + + @doc """ + Converts an `Explorer.Chain.Filecoin.IDAddress` to its string representation. + + ## Examples + + iex> address = %Explorer.Chain.Filecoin.IDAddress{value: <<193, 13>>} + iex> Explorer.Chain.Filecoin.IDAddress.to_string(address) + "f01729" + """ + @spec to_string(t()) :: String.t() + def to_string(%__MODULE__{} = address) do + address + |> to_native_address() + |> NativeAddress.to_string() + end + + defimpl String.Chars do + def to_string(address) do + @for.to_string(address) + end + end + + defimpl Poison.Encoder do + def encode(address, options) do + address + |> to_string() + |> BitString.encode(options) + end + end + + defimpl Jason.Encoder do + alias Jason.Encode + + def encode(address, opts) do + address + |> to_string() + |> Encode.string(opts) + end + end +end diff --git a/apps/explorer/lib/explorer/chain/filecoin/native_address.ex b/apps/explorer/lib/explorer/chain/filecoin/native_address.ex new file mode 100644 index 0000000000..aaeac0004d --- /dev/null +++ b/apps/explorer/lib/explorer/chain/filecoin/native_address.ex @@ -0,0 +1,408 @@ +defmodule Explorer.Chain.Filecoin.NativeAddress do + @moduledoc """ + Handles Filecoin addresses by parsing, validating, and converting them to and + from their binary representations. + + Addresses are encoded to binary according to the [Filecoin Address + spec](https://spec.filecoin.io/appendix/address/#section-appendix.address.validatechecksum). + Details about f4 addresses are provided in + [FIP-0048](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0048.md). + + Internally, f0/f1/f2/f3 addresses are stored as a binary with the following structure: + + |--------------------|---------| + | protocol indicator | payload | + |--------------------|---------| + | 1 byte | n bytes | + |--------------------|---------| + + 1. The first byte is the protocol indicator. The values are: + - `0` for f0 addresses + - `1` for f1 addresses + - `2` for f2 addresses + - `3` for f3 addresses + + 2. The remaining bytes are the payload. + + f4 addresses are stored as a binary with the following structure: + + |--------------------|----------|---------| + | protocol indicator | actor id | payload | + |--------------------|----------|---------| + | 1 byte | 1 byte | n bytes | + |--------------------|----------|---------| + + 1. The first byte is the protocol indicator. The value is `4`. + 2. The second byte is the actor id. + 3. The remaining bytes are the payload. + """ + + alias Explorer.Chain.Hash + alias Poison.Encoder.BitString + alias Varint.LEB128 + + use Ecto.Type + + defstruct ~w(protocol_indicator actor_id payload checksum)a + + @checksum_bytes_count 4 + + @protocol_indicator_bytes_count 1 + @max_protocol_indicator 2 ** (@protocol_indicator_bytes_count * Hash.bits_per_byte()) - 1 + + @min_address_string_length 3 + + # Payload sizes: + # f1 -- 20 bytes + # f2 -- 20 bytes + # f3 -- 48 bytes + @protocol_indicator_to_payload_byte_count %{ + 1 => 20, + # For some reason, specs tell that payload for f2 is a SHA256 hash, which is + # 32 bytes long. However, in practice, it is 20 bytes long... + # + # https://spec.filecoin.io/appendix/address/#section-appendix.address.protocol-2-actor + 2 => 20, + 3 => 48 + } + @standard_protocol_indicators Map.keys(@protocol_indicator_to_payload_byte_count) + + @type t :: %__MODULE__{ + protocol_indicator: non_neg_integer(), + actor_id: non_neg_integer() | nil, + payload: binary(), + checksum: binary() | nil + } + + @impl Ecto.Type + @spec type() :: :binary + def type, do: :binary + + defp network_prefix do + Atom.to_string(Application.get_env(:explorer, __MODULE__)[:network_prefix]) + end + + @doc """ + Casts `term` to `t:t/0`. + + If the term is already in `t:t/0`, then it is returned + + iex> Explorer.Chain.Filecoin.NativeAddress.cast( + ...> %Explorer.Chain.Filecoin.NativeAddress{ + ...> protocol_indicator: 0, + ...> actor_id: nil, + ...> payload: <<193, 13>>, + ...> checksum: nil + ...> } + ...> ) + { + :ok, + %Explorer.Chain.Filecoin.NativeAddress{ + protocol_indicator: 0, + actor_id: nil, + payload: <<193, 13>>, + checksum: nil + } + } + + If the term is a binary, then it is parsed to `t:t/0` + + iex> Explorer.Chain.Filecoin.NativeAddress.cast("f01729") + { + :ok, + %Explorer.Chain.Filecoin.NativeAddress{ + protocol_indicator: 0, + actor_id: nil, + payload: <<193, 13>>, + checksum: nil + } + } + + iex> Explorer.Chain.Filecoin.NativeAddress.cast("f01729") + { + :ok, + %Explorer.Chain.Filecoin.NativeAddress{ + protocol_indicator: 0, + actor_id: nil, + payload: <<193, 13>>, + checksum: nil + } + } + + iex> NativeAddress.cast("f410fabpafjfjgqkc3douo3yzfug5tq4bwfvuhsewxji") + { + :ok, + %Explorer.Chain.Filecoin.NativeAddress{ + protocol_indicator: 4, + actor_id: 10, + payload: <<0, 94, 2, 164, 169, 52, 20, 45, 141, 212, 118, 241, 146, 208, 221, 156, 56, 27, 22, 180>>, + checksum: <<60, 137, 107, 165>> + } + } + """ + @impl Ecto.Type + @spec cast(t() | String.t()) :: {:ok, t()} | :error + def cast(%__MODULE__{} = address), do: {:ok, address} + + def cast(address_string) when is_binary(address_string) do + network = network_prefix() + + with true <- String.length(address_string) >= @min_address_string_length, + ^network <> protocol_indicator_and_payload <- address_string, + {:ok, address} <- cast_protocol_indicator_and_payload(protocol_indicator_and_payload), + :ok <- verify_checksum(address) do + {:ok, address} + else + _ -> + :error + end + end + + defp cast_protocol_indicator_and_payload("0" <> id_string) do + id_string + |> Integer.parse() + |> case do + {id, ""} when is_integer(id) and id >= 0 -> + payload = LEB128.encode(id) + + {:ok, + %__MODULE__{ + protocol_indicator: 0, + actor_id: nil, + payload: payload, + checksum: nil + }} + + _ -> + :error + end + end + + defp cast_protocol_indicator_and_payload("4" <> rest) do + with [actor_id_string, base32_digits] <- String.split(rest, "f", parts: 2), + {actor_id, ""} when is_integer(actor_id) <- Integer.parse(actor_id_string), + {:ok, {payload, checksum}} <- cast_base32_digits(base32_digits) do + {:ok, + %__MODULE__{ + protocol_indicator: 4, + actor_id: actor_id, + payload: payload, + checksum: checksum + }} + else + _ -> :error + end + end + + defp cast_protocol_indicator_and_payload(protocol_indicator_and_payload) do + with {protocol_indicator_string, base32_digits} <- + String.split_at( + protocol_indicator_and_payload, + 1 + ), + {protocol_indicator, ""} when protocol_indicator in @standard_protocol_indicators <- + Integer.parse(protocol_indicator_string), + {:ok, byte_count} <- + Map.fetch( + @protocol_indicator_to_payload_byte_count, + protocol_indicator + ), + {:ok, {payload, checksum}} <- cast_base32_digits(base32_digits, byte_count) do + {:ok, + %__MODULE__{ + protocol_indicator: protocol_indicator, + actor_id: nil, + payload: payload, + checksum: checksum + }} + else + _ -> :error + end + end + + defp cast_base32_digits(digits) do + with {:ok, bytes} <- Base.decode32(digits, case: :lower, padding: false), + << + payload::binary-size(byte_size(bytes) - @checksum_bytes_count), + checksum::binary-size(@checksum_bytes_count) + >> <- bytes do + {:ok, {payload, checksum}} + else + _ -> :error + end + end + + defp cast_base32_digits(digits, expected_bytes_count) do + with {:ok, {payload, checksum}} <- cast_base32_digits(digits), + true <- byte_size(payload) == expected_bytes_count do + {:ok, {payload, checksum}} + else + _ -> :error + end + end + + @doc """ + Dumps the address to `:binary` (`bytea`) representation format used in + database. + """ + @impl Ecto.Type + @spec dump(t()) :: {:ok, binary()} | :error + def dump(%__MODULE__{protocol_indicator: 4, actor_id: actor_id, payload: payload}) + when is_integer(actor_id) and + is_binary(payload) and + actor_id >= 0 and + actor_id <= @max_protocol_indicator do + {:ok, <<4, actor_id, payload::binary>>} + end + + def dump(%__MODULE__{protocol_indicator: protocol_indicator, payload: payload}) + when is_integer(protocol_indicator) and + is_binary(payload) and + protocol_indicator >= 0 and + protocol_indicator <= @max_protocol_indicator do + {:ok, <>} + end + + def dump(_), do: :error + + @doc """ + Loads the address from `:binary` representation used in database. + """ + @impl Ecto.Type + @spec load(binary()) :: {:ok, t()} | :error + def load(<> = bytes) do + case protocol_indicator do + 0 -> + {:ok, + %__MODULE__{ + protocol_indicator: 0, + actor_id: nil, + payload: rest, + checksum: nil + }} + + 4 -> + checksum = to_checksum(bytes) + <> = rest + + {:ok, + %__MODULE__{ + protocol_indicator: 4, + actor_id: actor_id, + payload: payload, + checksum: checksum + }} + + protocol_indicator when protocol_indicator in @standard_protocol_indicators -> + checksum = to_checksum(bytes) + + {:ok, + %__MODULE__{ + protocol_indicator: protocol_indicator, + actor_id: nil, + payload: rest, + checksum: checksum + }} + + _ -> + :error + end + end + + def load(_), do: :error + + @doc """ + Converts the address to a string representation. + + iex> Explorer.Chain.Filecoin.NativeAddress.to_string( + ...> %Explorer.Chain.Filecoin.NativeAddress{ + ...> protocol_indicator: 0, + ...> actor_id: nil, + ...> payload: <<193, 13>>, + ...> checksum: nil + ...> } + ...> ) + "f01729" + + iex> Explorer.Chain.Filecoin.NativeAddress.to_string( + ...> %Explorer.Chain.Filecoin.NativeAddress{ + ...> protocol_indicator: 4, + ...> actor_id: 10, + ...> payload: <<0, 94, 2, 164, 169, 52, 20, 45, 141, 212, 118, 241, 146, 208, 221, 156, 56, 27, 22, 180>>, + ...> checksum: <<60, 137, 107, 165>> + ...> } + ...> ) + "f410fabpafjfjgqkc3douo3yzfug5tq4bwfvuhsewxji" + """ + @spec to_string(t) :: String.t() + def to_string(%__MODULE__{protocol_indicator: 0, payload: payload}) do + {id, <<>>} = LEB128.decode(payload) + network_prefix() <> "0" <> Integer.to_string(id) + end + + @spec to_string(t) :: String.t() + def to_string(%__MODULE__{ + protocol_indicator: protocol_indicator, + payload: payload, + actor_id: actor_id, + checksum: checksum + }) do + payload_with_checksum = + Base.encode32( + payload <> checksum, + case: :lower, + padding: false + ) + + protocol_indicator_part = + protocol_indicator + |> case do + indicator when indicator in @standard_protocol_indicators -> + Integer.to_string(indicator) + + 4 -> + "4" <> Integer.to_string(actor_id) <> "f" + end + + network_prefix() <> protocol_indicator_part <> payload_with_checksum + end + + defp verify_checksum(%__MODULE__{protocol_indicator: 0, checksum: nil}), do: :ok + + defp verify_checksum(%__MODULE__{checksum: checksum} = address) + when not is_nil(checksum) do + with {:ok, bytes} <- dump(address), + ^checksum <- to_checksum(bytes) do + :ok + else + _ -> :error + end + end + + defp to_checksum(bytes), + do: Blake2.hash2b(bytes, @checksum_bytes_count) + + defimpl String.Chars do + def to_string(hash) do + @for.to_string(hash) + end + end + + defimpl Poison.Encoder do + def encode(hash, options) do + hash + |> to_string() + |> BitString.encode(options) + end + end + + defimpl Jason.Encoder do + alias Jason.Encode + + def encode(hash, opts) do + hash + |> to_string() + |> Encode.string(opts) + end + end +end diff --git a/apps/explorer/lib/explorer/chain/filecoin/pending_address_operation.ex b/apps/explorer/lib/explorer/chain/filecoin/pending_address_operation.ex new file mode 100644 index 0000000000..0fa166e0fb --- /dev/null +++ b/apps/explorer/lib/explorer/chain/filecoin/pending_address_operation.ex @@ -0,0 +1,73 @@ +defmodule Explorer.Chain.Filecoin.PendingAddressOperation do + @moduledoc """ + Tracks an address that is pending for fetching of filecoin address info. + """ + + use Explorer.Schema + + import Explorer.Chain, only: [add_fetcher_limit: 2] + alias Explorer.Chain.{Address, Hash} + alias Explorer.Repo + + @http_error_codes 400..526 + + @optional_attrs ~w(http_status_code)a + @required_attrs ~w(address_hash)a + + @attrs @optional_attrs ++ @required_attrs + + @typedoc """ + * `address_hash` - the hash of the address that is pending to be fetched. + * `http_status_code` - the unsuccessful (non-200) http code returned by Beryx + API if the fetcher failed to fetch the address. + """ + @primary_key false + typed_schema "filecoin_pending_address_operations" do + belongs_to(:address, Address, + foreign_key: :address_hash, + references: :hash, + type: Hash.Address, + primary_key: true + ) + + field(:http_status_code, :integer) + + timestamps() + end + + @spec changeset( + Explorer.Chain.Filecoin.PendingAddressOperation.t(), + :invalid | %{optional(:__struct__) => none(), optional(atom() | binary()) => any()} + ) :: Ecto.Changeset.t() + def changeset(%__MODULE__{} = pending_ops, attrs) do + pending_ops + |> cast(attrs, @attrs) + |> validate_required(@required_attrs) + |> foreign_key_constraint(:address_hash, name: :filecoin_pending_address_operations_address_hash_fkey) + |> unique_constraint(:address_hash, name: :filecoin_pending_address_operations_pkey) + |> validate_inclusion(:http_status_code, @http_error_codes) + end + + @doc """ + Returns a stream of pending operations. + """ + @spec stream( + initial :: accumulator, + reducer :: (entry :: term(), accumulator -> accumulator), + limited? :: boolean() + ) :: {:ok, accumulator} + when accumulator: term() + def stream(initial, reducer, limited? \\ false) + when is_function(reducer, 2) do + query = + from( + op in __MODULE__, + select: op, + order_by: [desc: op.address_hash] + ) + + query + |> add_fetcher_limit(limited?) + |> Repo.stream_reduce(initial, reducer) + end +end diff --git a/apps/explorer/lib/explorer/chain/hash.ex b/apps/explorer/lib/explorer/chain/hash.ex index 8bac3ff442..255861c0d6 100644 --- a/apps/explorer/lib/explorer/chain/hash.ex +++ b/apps/explorer/lib/explorer/chain/hash.ex @@ -121,7 +121,7 @@ defmodule Explorer.Chain.Hash do """ @spec to_integer(t()) :: pos_integer() def to_integer(%__MODULE__{byte_count: byte_count, bytes: bytes}) do - <> = bytes + <> = bytes integer end diff --git a/apps/explorer/lib/explorer/chain/import/runner/addresses.ex b/apps/explorer/lib/explorer/chain/import/runner/addresses.ex index 23a2e7a1d0..fe6cea4f7d 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/addresses.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/addresses.ex @@ -3,14 +3,16 @@ defmodule Explorer.Chain.Import.Runner.Addresses do Bulk imports `t:Explorer.Chain.Address.t/0`. """ - require Ecto.Query + import Ecto.Query, only: [from: 2] + import Explorer.Chain.Import.Runner.Helper, only: [chain_type_dependent_import: 3] alias Ecto.{Multi, Repo} - alias Explorer.Chain.{Address, Hash, Import, Transaction} + alias Explorer.Chain.Filecoin.PendingAddressOperation, as: FilecoinPendingAddressOperation alias Explorer.Chain.Import.Runner + alias Explorer.Chain.{Address, Hash, Import, Transaction} alias Explorer.Prometheus.Instrumenter - import Ecto.Query, only: [from: 2] + require Ecto.Query @behaviour Import.Runner @@ -98,6 +100,21 @@ defmodule Explorer.Chain.Import.Runner.Addresses do :created_address_code_indexed_at_transactions ) end) + |> chain_type_dependent_import( + :filecoin, + &Multi.run( + &1, + :filecoin_pending_address_operations, + fn repo, _ -> + Instrumenter.block_import_stage_runner( + fn -> filecoin_pending_address_operations(repo, ordered_changes_list, insert_options) end, + :addresses, + :addresses, + :filecoin_pending_address_operations + ) + end + ) + ) end @impl Import.Runner @@ -261,4 +278,23 @@ defmodule Explorer.Chain.Import.Runner.Addresses do end end end + + defp filecoin_pending_address_operations(repo, addresses, %{timeout: timeout, timestamps: timestamps}) do + ordered_addresses = + addresses + |> Enum.map(&%{address_hash: &1.hash}) + |> Enum.sort_by(& &1.address_hash) + |> Enum.dedup_by(& &1.address_hash) + + Import.insert_changes_list( + repo, + ordered_addresses, + conflict_target: :address_hash, + on_conflict: :nothing, + for: FilecoinPendingAddressOperation, + returning: true, + timeout: timeout, + timestamps: timestamps + ) + end end diff --git a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex index 0af2896b2c..3043ce24ec 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex @@ -6,6 +6,7 @@ defmodule Explorer.Chain.Import.Runner.Blocks do require Ecto.Query import Ecto.Query, only: [from: 2, where: 3, subquery: 1] + import Explorer.Chain.Import.Runner.Helper, only: [chain_type_dependent_import: 3] alias Ecto.{Changeset, Multi, Repo} @@ -228,14 +229,6 @@ defmodule Explorer.Chain.Import.Runner.Blocks do @impl Runner def timeout, do: @timeout - def chain_type_dependent_import(multi, chain_type, multi_run) do - if Application.get_env(:explorer, :chain_type) == chain_type do - multi_run.(multi) - else - multi - end - end - defp fork_transactions(%{ repo: repo, timeout: timeout, diff --git a/apps/explorer/lib/explorer/chain/import/runner/helper.ex b/apps/explorer/lib/explorer/chain/import/runner/helper.ex new file mode 100644 index 0000000000..6fd5f53e85 --- /dev/null +++ b/apps/explorer/lib/explorer/chain/import/runner/helper.ex @@ -0,0 +1,22 @@ +defmodule Explorer.Chain.Import.Runner.Helper do + @moduledoc """ + Provides utility functions for the chain import runners. + """ + + @doc """ + Executes the import function if the configured chain type matches the + specified `chain_type`. + """ + @spec chain_type_dependent_import( + Ecto.Multi.t(), + chain_type :: atom(), + (Ecto.Multi.t() -> Ecto.Multi.t()) + ) :: Ecto.Multi.t() + def chain_type_dependent_import(multi, chain_type, multi_run) do + if Application.get_env(:explorer, :chain_type) == chain_type do + multi_run.(multi) + else + multi + end + end +end diff --git a/apps/explorer/lib/explorer/migrator/filecoin_pending_address_operations.ex b/apps/explorer/lib/explorer/migrator/filecoin_pending_address_operations.ex new file mode 100644 index 0000000000..47254d8805 --- /dev/null +++ b/apps/explorer/lib/explorer/migrator/filecoin_pending_address_operations.ex @@ -0,0 +1,71 @@ +defmodule Explorer.Migrator.FilecoinPendingAddressOperations do + @moduledoc """ + Creates a pending address operation for each address missing Filecoin address + information, specifically when `filecoin_id`, `filecoin_robust`, and + `filecoin_actor_type` are `nil`. + """ + + use Explorer.Migrator.FillingMigration + + import Ecto.Query + + alias Explorer.Chain.{Address, Filecoin.PendingAddressOperation, Import} + alias Explorer.Migrator.FillingMigration + alias Explorer.Repo + + @migration_name "filecoin_pending_address_operations" + + @impl FillingMigration + def migration_name, do: @migration_name + + @impl FillingMigration + def last_unprocessed_identifiers(state) do + limit = batch_size() * concurrency() + + ids = + unprocessed_data_query() + |> select([address], address.hash) + |> limit(^limit) + |> Repo.all(timeout: :infinity) + + {ids, state} + end + + @impl FillingMigration + def unprocessed_data_query do + from( + address in Address, + left_join: op in PendingAddressOperation, + on: address.hash == op.address_hash, + where: + is_nil(address.filecoin_id) and + is_nil(address.filecoin_robust) and + is_nil(address.filecoin_actor_type) and + is_nil(op.address_hash), + order_by: [asc: address.hash] + ) + end + + @impl FillingMigration + def update_batch(ordered_address_hashes) do + ordered_pending_operations = + Enum.map( + ordered_address_hashes, + &%{address_hash: &1} + ) + + Import.insert_changes_list( + Repo, + ordered_pending_operations, + conflict_target: :address_hash, + on_conflict: :nothing, + for: PendingAddressOperation, + returning: true, + timeout: :infinity, + timestamps: Import.timestamps() + ) + end + + @impl FillingMigration + def update_cache, do: :ok +end diff --git a/apps/explorer/mix.exs b/apps/explorer/mix.exs index 56345b2e1d..d772ad4c19 100644 --- a/apps/explorer/mix.exs +++ b/apps/explorer/mix.exs @@ -122,7 +122,9 @@ defmodule Explorer.Mixfile do {:logger_json, "~> 5.1"}, {:typed_ecto_schema, "~> 0.4.1", runtime: false}, {:ueberauth, "~> 0.7"}, - {:recon, "~> 2.5"} + {:recon, "~> 2.5"}, + {:varint, "~> 1.4"}, + {:blake2, "~> 1.0"} ] end diff --git a/apps/explorer/priv/filecoin/migrations/20240801134142_create_pending_address_operations.exs b/apps/explorer/priv/filecoin/migrations/20240801134142_create_pending_address_operations.exs new file mode 100644 index 0000000000..7da899939b --- /dev/null +++ b/apps/explorer/priv/filecoin/migrations/20240801134142_create_pending_address_operations.exs @@ -0,0 +1,23 @@ +defmodule Explorer.Repo.Filecoin.Migrations.CreatePendingAddressOperations do + use Ecto.Migration + + def change do + create table(:filecoin_pending_address_operations, primary_key: false) do + add( + :address_hash, + references( + :addresses, + column: :hash, + type: :bytea, + on_delete: :delete_all + ), + null: false, + primary_key: true + ) + + add(:http_status_code, :smallint) + + timestamps() + end + end +end diff --git a/apps/explorer/priv/filecoin/migrations/20240807134138_add_chain_type_fields_to_address.exs b/apps/explorer/priv/filecoin/migrations/20240807134138_add_chain_type_fields_to_address.exs new file mode 100644 index 0000000000..378e3b24c0 --- /dev/null +++ b/apps/explorer/priv/filecoin/migrations/20240807134138_add_chain_type_fields_to_address.exs @@ -0,0 +1,11 @@ +defmodule Explorer.Repo.Filecoin.Migrations.AddChainTypeFieldsToAddress do + use Ecto.Migration + + def change do + alter table(:addresses) do + add(:filecoin_id, :bytea) + add(:filecoin_robust, :bytea) + add(:filecoin_actor_type, :smallint) + end + end +end diff --git a/apps/explorer/test/explorer/chain/filecoin/native_address_test.exs b/apps/explorer/test/explorer/chain/filecoin/native_address_test.exs new file mode 100644 index 0000000000..28368261e4 --- /dev/null +++ b/apps/explorer/test/explorer/chain/filecoin/native_address_test.exs @@ -0,0 +1,175 @@ +defmodule Explorer.Chain.Filecoin.NativeAddressTest do + use ExUnit.Case, async: true + + alias Explorer.Chain.Hash + alias Explorer.Chain.Hash.Address + alias Explorer.Chain.Filecoin.{NativeAddress, IDAddress} + + doctest NativeAddress + doctest IDAddress + + @doc """ + The following test cases are taken from the filecoin spec: + https://spec.filecoin.io/appendix/address/#section-appendix.address.test-vectors + + The key is the address and the value is the hex-encoded binary representation + of the address in the database. + """ + # cspell:disable + @test_cases %{ + "f00" => "0000", + "f0150" => "009601", + "f01024" => "008008", + "f01729" => "00c10d", + "f018446744073709551615" => "00ffffffffffffffffff01", + "f17uoq6tp427uzv7fztkbsnn64iwotfrristwpryy" => "01fd1d0f4dfcd7e99afcb99a8326b7dc459d32c628", + "f1xcbgdhkgkwht3hrrnui3jdopeejsoatkzmoltqy" => "01b882619d46558f3d9e316d11b48dcf211327026a", + "f1xtwapqc6nh4si2hcwpr3656iotzmlwumogqbuaa" => "01bcec07c05e69f92468e2b3e3bf77c874f2c5da8c", + "f1wbxhu3ypkuo6eyp6hjx6davuelxaxrvwb2kuwva" => "01b06e7a6f0f551de261fe3a6fe182b422ee0bc6b6", + "f12fiakbhe2gwd5cnmrenekasyn6v5tnaxaqizq6a" => "01d1500504e4d1ac3e89ac891a4502586fabd9b417", + "f24vg6ut43yw2h2jqydgbg2xq7x6f4kub3bg6as6i" => "02e54dea4f9bc5b47d261819826d5e1fbf8bc5503b", + "f25nml2cfbljvn4goqtclhifepvfnicv6g7mfmmvq" => "02eb58bd08a15a6ade19d0989674148fa95a8157c6", + "f2nuqrg7vuysaue2pistjjnt3fadsdzvyuatqtfei" => "026d21137eb4c4814269e894d296cf6500e43cd714", + "f24dd4ox4c2vpf5vk5wkadgyyn6qtuvgcpxxon64a" => "02e0c7c75f82d55e5ed55db28033630df4274a984f", + "f2gfvuyh7v2sx3patm5k23wdzmhyhtmqctasbr23y" => "02316b4c1ff5d4afb7826ceab5bb0f2c3e0f364053", + "f3vvmn62lofvhjd2ugzca6sof2j2ubwok6cj4xxbfzz4yuxfkgobpihhd2thlanmsh3w2ptld2gqkn2jvlss4a" => + "03ad58df696e2d4e91ea86c881e938ba4ea81b395e12797b84b9cf314b9546705e839c7a99d606b247ddb4f9ac7a3414dd", + "f3wmuu6crofhqmm3v4enos73okk2l366ck6yc4owxwbdtkmpk42ohkqxfitcpa57pjdcftql4tojda2poeruwa" => + "03b3294f0a2e29e0c66ebc235d2fedca5697bf784af605c75af608e6a63d5cd38ea85ca8989e0efde9188b382f9372460d", + "f3s2q2hzhkpiknjgmf4zq3ejab2rh62qbndueslmsdzervrhapxr7dftie4kpnpdiv2n6tvkr743ndhrsw6d3a" => + "0396a1a3e4ea7a14d49985e661b22401d44fed402d1d0925b243c923589c0fbc7e32cd04e29ed78d15d37d3aaa3fe6da33", + "f3q22fijmmlckhl56rn5nkyamkph3mcfu5ed6dheq53c244hfmnq2i7efdma3cj5voxenwiummf2ajlsbxc65a" => + "0386b454258c589475f7d16f5aac018a79f6c1169d20fc33921dd8b5ce1cac6c348f90a3603624f6aeb91b64518c2e8095", + "f3u5zgwa4ael3vuocgc5mfgygo4yuqocrntuuhcklf4xzg5tcaqwbyfabxetwtj4tsam3pbhnwghyhijr5mixa" => + "03a7726b038022f75a384617585360cee629070a2d9d28712965e5f26ecc40858382803724ed34f2720336f09db631f074" + } + + # cspell:enable + + describe "cast/1" do + test "parses f0, f1, f2, f3 addresses from spec test vectors" do + for {address, hex_string} <- @test_cases do + {protocol_indicator_hex, payload} = String.split_at(hex_string, 2) + protocol_indicator = String.to_integer(protocol_indicator_hex, 16) + payload = Base.decode16!(payload, case: :lower) + + assert {:ok, + %NativeAddress{ + protocol_indicator: ^protocol_indicator, + actor_id: nil, + payload: ^payload + }} = NativeAddress.cast(address) + end + end + + test "parses f4 addresses" do + address = "f410fabpafjfjgqkc3douo3yzfug5tq4bwfvuhsewxji" + {:ok, evm_address} = Address.cast("0x005E02A4A934142D8DD476F192D0DD9C381B16B4") + evm_address_bytes = evm_address.bytes + + assert {:ok, + %NativeAddress{ + protocol_indicator: 4, + actor_id: 10, + payload: ^evm_address_bytes + }} = NativeAddress.cast(address) + end + end + + describe "dump/1" do + test "encodes f0, f1, f2, f3 addresses to bytes" do + for {address, hex_string} <- @test_cases do + bytes = Base.decode16!(hex_string, case: :lower) + + assert {:ok, ^bytes} = + address + |> NativeAddress.cast() + |> elem(1) + |> NativeAddress.dump() + end + end + + test "converts f4 addresses" do + address = "f410fabpafjfjgqkc3douo3yzfug5tq4bwfvuhsewxji" + {:ok, evm_address} = Address.cast("0x005E02A4A934142D8DD476F192D0DD9C381B16B4") + bytes = <<4, 10, evm_address.bytes::binary>> + + assert {:ok, ^bytes} = + address + |> NativeAddress.cast() + |> elem(1) + |> NativeAddress.dump() + end + end + + describe "load/1" do + test "decodes f0, f1, f2, f3 addresses from bytes" do + for {address, hex_string} <- Map.values(@test_cases) do + {protocol_indicator_hex, payload_hex} = String.split_at(hex_string, 2) + protocol_indicator = String.to_integer(protocol_indicator_hex, 16) + payload = Base.decode16!(payload_hex, case: :lower) + + assert {:ok, + %NativeAddress{ + protocol_indicator: ^protocol_indicator, + actor_id: nil, + payload: ^payload + }} = + address + |> NativeAddress.cast() + |> elem(1) + |> NativeAddress.dump() + |> elem(1) + |> NativeAddress.load() + end + end + + test "decodes f4 addresses" do + address = "f410fabpafjfjgqkc3douo3yzfug5tq4bwfvuhsewxji" + {:ok, %Hash{bytes: payload}} = Address.cast("0x005E02A4A934142D8DD476F192D0DD9C381B16B4") + + assert {:ok, + %NativeAddress{ + protocol_indicator: 4, + actor_id: 10, + payload: ^payload + }} = + address + |> NativeAddress.cast() + |> elem(1) + |> NativeAddress.dump() + |> elem(1) + |> NativeAddress.load() + end + end + + describe "to_string/1" do + test "converts f0, f1, f2, f3 addresses to string" do + for {address, _} <- @test_cases do + assert ^address = + address + |> NativeAddress.cast() + |> elem(1) + |> NativeAddress.dump() + |> elem(1) + |> NativeAddress.load() + |> elem(1) + |> NativeAddress.to_string() + end + end + + test "converts f4 addresses to string" do + address = "f410fabpafjfjgqkc3douo3yzfug5tq4bwfvuhsewxji" + + assert ^address = + address + |> NativeAddress.cast() + |> elem(1) + |> NativeAddress.dump() + |> elem(1) + |> NativeAddress.load() + |> elem(1) + |> NativeAddress.to_string() + end + end +end diff --git a/apps/indexer/lib/indexer/application.ex b/apps/indexer/lib/indexer/application.ex index 2a688a1a6f..78d05b37b0 100644 --- a/apps/indexer/lib/indexer/application.ex +++ b/apps/indexer/lib/indexer/application.ex @@ -12,12 +12,30 @@ defmodule Indexer.Application do alias Indexer.Fetcher.OnDemand.TokenTotalSupply, as: TokenTotalSupplyOnDemand alias Indexer.Memory - alias Indexer.Prometheus.PendingBlockOperationsCollector alias Prometheus.Registry + @default_prometheus_collectors [ + Indexer.Prometheus.Collector.PendingBlockOperations + ] + + case Application.compile_env(:explorer, :chain_type) do + :filecoin -> + @chain_type_prometheus_collectors [ + Indexer.Prometheus.Collector.FilecoinPendingAddressOperations + ] + + _ -> + @chain_type_prometheus_collectors [] + end + + @prometheus_collectors @default_prometheus_collectors ++ + @chain_type_prometheus_collectors + @impl Application def start(_type, _args) do - Registry.register_collector(PendingBlockOperationsCollector) + for collector <- @prometheus_collectors do + Registry.register_collector(collector) + end memory_monitor_options = case Application.get_env(:indexer, :memory_limit) do diff --git a/apps/indexer/lib/indexer/block/catchup/fetcher.ex b/apps/indexer/lib/indexer/block/catchup/fetcher.ex index 180ddf37ec..7f3d32ecf5 100644 --- a/apps/indexer/lib/indexer/block/catchup/fetcher.ex +++ b/apps/indexer/lib/indexer/block/catchup/fetcher.ex @@ -14,6 +14,7 @@ defmodule Indexer.Block.Catchup.Fetcher do async_import_celo_epoch_block_operations: 2, async_import_coin_balances: 2, async_import_created_contract_codes: 2, + async_import_filecoin_addresses_info: 2, async_import_internal_transactions: 2, async_import_replaced_transactions: 2, async_import_token_balances: 2, @@ -141,6 +142,7 @@ defmodule Indexer.Block.Catchup.Fetcher do async_import_token_instances(imported) async_import_blobs(imported, realtime?) async_import_celo_epoch_block_operations(imported, realtime?) + async_import_filecoin_addresses_info(imported, realtime?) end defp stream_fetch_and_import(state, ranges) do diff --git a/apps/indexer/lib/indexer/block/fetcher.ex b/apps/indexer/lib/indexer/block/fetcher.ex index 03082037a2..b8a4d716b2 100644 --- a/apps/indexer/lib/indexer/block/fetcher.ex +++ b/apps/indexer/lib/indexer/block/fetcher.ex @@ -11,15 +11,17 @@ defmodule Indexer.Block.Fetcher do alias EthereumJSONRPC.{Blocks, FetchedBeneficiaries} alias Explorer.Chain - alias Explorer.Chain.{Address, Block, Hash, Import, Transaction, Wei} alias Explorer.Chain.Block.Reward alias Explorer.Chain.Cache.Blocks, as: BlocksCache alias Explorer.Chain.Cache.{Accounts, BlockNumber, Transactions, Uncles} + alias Explorer.Chain.Filecoin.PendingAddressOperation, as: FilecoinPendingAddressOperation + alias Explorer.Chain.{Address, Block, Hash, Import, Transaction, Wei} alias Indexer.Block.Fetcher.Receipts alias Indexer.Fetcher.Celo.EpochBlockOperations, as: CeloEpochBlockOperations alias Indexer.Fetcher.Celo.EpochLogs, as: CeloEpochLogs alias Indexer.Fetcher.CoinBalance.Catchup, as: CoinBalanceCatchup alias Indexer.Fetcher.CoinBalance.Realtime, as: CoinBalanceRealtime + alias Indexer.Fetcher.Filecoin.AddressInfo, as: FilecoinAddressInfo alias Indexer.Fetcher.PolygonZkevm.BridgeL1Tokens, as: PolygonZkevmBridgeL1Tokens alias Indexer.Fetcher.TokenInstance.Realtime, as: TokenInstanceRealtime @@ -511,6 +513,14 @@ defmodule Indexer.Block.Fetcher do def async_import_celo_epoch_block_operations(_, _), do: :ok + def async_import_filecoin_addresses_info(%{addresses: addresses}, realtime?) do + addresses + |> Enum.map(&%FilecoinPendingAddressOperation{address_hash: &1.hash}) + |> FilecoinAddressInfo.async_fetch(realtime?) + end + + def async_import_filecoin_addresses_info(_, _), do: :ok + defp block_reward_errors_to_block_numbers(block_reward_errors) when is_list(block_reward_errors) do Enum.map(block_reward_errors, &block_reward_error_to_block_number/1) end diff --git a/apps/indexer/lib/indexer/block/realtime/fetcher.ex b/apps/indexer/lib/indexer/block/realtime/fetcher.ex index 10d6fbafc2..a2e7716ac1 100644 --- a/apps/indexer/lib/indexer/block/realtime/fetcher.ex +++ b/apps/indexer/lib/indexer/block/realtime/fetcher.ex @@ -17,6 +17,7 @@ defmodule Indexer.Block.Realtime.Fetcher do async_import_block_rewards: 2, async_import_celo_epoch_block_operations: 2, async_import_created_contract_codes: 2, + async_import_filecoin_addresses_info: 2, async_import_internal_transactions: 2, async_import_polygon_zkevm_bridge_l1_tokens: 1, async_import_realtime_coin_balances: 1, @@ -467,5 +468,6 @@ defmodule Indexer.Block.Realtime.Fetcher do async_import_blobs(imported, realtime?) async_import_polygon_zkevm_bridge_l1_tokens(imported) async_import_celo_epoch_block_operations(imported, realtime?) + async_import_filecoin_addresses_info(imported, realtime?) end end diff --git a/apps/indexer/lib/indexer/fetcher/filecoin/address_info.ex b/apps/indexer/lib/indexer/fetcher/filecoin/address_info.ex new file mode 100644 index 0000000000..83e5f7cdd1 --- /dev/null +++ b/apps/indexer/lib/indexer/fetcher/filecoin/address_info.ex @@ -0,0 +1,215 @@ +defmodule Indexer.Fetcher.Filecoin.AddressInfo do + @moduledoc """ + A task for fetching Filecoin addresses info in the Address table using the + Beryx API. + + Due to the lack of batch support in the API, addresses are fetched + individually, making this fetching an expensive operation. + """ + use Indexer.Fetcher, restart: :permanent + use Spandex.Decorators + + alias Ecto.Multi + alias Explorer.Chain.{Address, Filecoin.PendingAddressOperation} + alias Explorer.Repo + alias Indexer.Fetcher.Filecoin.AddressInfo.Supervisor, as: FilecoinAddressInfoSupervisor + alias Indexer.Fetcher.Filecoin.BeryxAPI + alias Indexer.{BufferedTask, Tracer} + + @http_error_codes 400..526 + + @batch_size 1 + + @behaviour BufferedTask + + require Logger + + @doc """ + Asynchronously fetches filecoin addresses info + """ + @spec async_fetch([PendingAddressOperation.t()], boolean(), integer()) :: :ok + def async_fetch(pending_operations, realtime?, timeout \\ 5000) + when is_list(pending_operations) do + if FilecoinAddressInfoSupervisor.disabled?() do + :ok + else + unique_operations = + Enum.uniq_by( + pending_operations, + &to_string(&1.address_hash) + ) + + BufferedTask.buffer(__MODULE__, unique_operations, realtime?, timeout) + end + end + + @doc false + @spec child_spec([...]) :: Supervisor.child_spec() + def child_spec([init_options, gen_server_options]) do + merged_init_opts = + defaults() + |> Keyword.merge(init_options) + |> Keyword.put(:state, nil) + + Supervisor.child_spec( + {BufferedTask, [{__MODULE__, merged_init_opts}, gen_server_options]}, + id: __MODULE__ + ) + end + + @doc false + @impl BufferedTask + def init(initial, reducer, _) do + {:ok, final} = + PendingAddressOperation.stream( + initial, + fn op, acc -> reducer.(op, acc) end + ) + + final + end + + @doc false + @spec defaults() :: Keyword.t() + def defaults do + env = Application.get_env(:indexer, __MODULE__) + + [ + poll: false, + flush_interval: :timer.seconds(30), + max_concurrency: env[:concurrency], + max_batch_size: @batch_size, + task_supervisor: __MODULE__.TaskSupervisor, + metadata: [fetcher: :filecoin_address_info] + ] + end + + @doc """ + Fetches the Filecoin address info for the given pending operation. + """ + @impl BufferedTask + @decorate trace( + name: "fetch", + resource: "Indexer.Fetcher.InternalTransaction.run/2", + service: :indexer, + tracer: Tracer + ) + @spec run([Explorer.Chain.Filecoin.PendingAddressOperation.t(), ...], any()) :: :ok | :retry + def run([pending_operation], _state) do + fetch_and_update(pending_operation) + end + + @spec fetch_and_update(PendingAddressOperation.t()) :: :ok | :retry + defp fetch_and_update(%PendingAddressOperation{address_hash: address_hash} = operation) do + with {:ok, new_params} <- fetch_address_info_using_beryx_api(operation), + {:ok, _} <- update_address_and_remove_pending_operation(operation, new_params) do + Logger.debug("Fetched Filecoin address info for: #{to_string(address_hash)}") + :ok + else + _ -> + Logger.error("Could not fetch Filecoin address info: #{to_string(address_hash)}") + :retry + end + end + + @spec update_address_and_remove_pending_operation( + PendingAddressOperation.t(), + %{ + filecoin_id: String.t(), + filecoin_robust: String.t(), + filecoin_actor_type: String.t() + } + ) :: + {:ok, PendingAddressOperation.t()} + | {:error, Ecto.Changeset.t()} + | Ecto.Multi.failure() + defp update_address_and_remove_pending_operation( + %PendingAddressOperation{} = operation, + new_address_params + ) do + Multi.new() + |> Multi.run( + :acquire_address, + fn repo, _ -> + case repo.get_by( + Address, + [hash: operation.address_hash], + lock: "FOR UPDATE" + ) do + nil -> {:error, :not_found} + address -> {:ok, address} + end + end + ) + |> Multi.run( + :acquire_pending_address_operation, + fn repo, _ -> + case repo.get_by( + PendingAddressOperation, + [address_hash: operation.address_hash], + lock: "FOR UPDATE" + ) do + nil -> {:error, :not_found} + pending_operation -> {:ok, pending_operation} + end + end + ) + |> Multi.run( + :update_address, + fn repo, %{acquire_address: address} -> + address + |> Address.changeset(new_address_params) + |> repo.update() + end + ) + |> Multi.run( + :delete_pending_operation, + fn repo, %{acquire_pending_address_operation: operation} -> + repo.delete(operation) + end + ) + |> Repo.transaction() + end + + @spec fetch_address_info_using_beryx_api(PendingAddressOperation.t()) :: + {:ok, + %{ + filecoin_id: String.t(), + filecoin_robust: String.t(), + filecoin_actor_type: String.t() + }} + | :error + defp fetch_address_info_using_beryx_api(%PendingAddressOperation{} = operation) do + with {:ok, body_json} <- operation.address_hash |> to_string() |> BeryxAPI.fetch_account_info(), + {:ok, id_address_string} <- Map.fetch(body_json, "short"), + {:ok, robust_address_string} <- Map.fetch(body_json, "robust"), + {:ok, actor_type_string} <- Map.fetch(body_json, "actor_type") do + {:ok, + %{ + filecoin_id: id_address_string, + filecoin_robust: robust_address_string, + filecoin_actor_type: actor_type_string + }} + else + {:error, status_code, %{"error" => reason}} when status_code in @http_error_codes -> + Logger.error("Beryx API returned error code #{status_code} with reason: #{reason}") + + operation + |> PendingAddressOperation.changeset(%{http_status_code: status_code}) + |> Repo.update() + |> case do + {:ok, _} -> + Logger.info("Updated pending operation with error status code") + + {:error, changeset} -> + Logger.error("Could not update pending operation with error status code: #{inspect(changeset)}") + end + + :error + + error -> + Logger.error("Error processing Beryx API response: #{inspect(error)}") + :error + end + end +end diff --git a/apps/indexer/lib/indexer/fetcher/filecoin/beryx_api.ex b/apps/indexer/lib/indexer/fetcher/filecoin/beryx_api.ex new file mode 100644 index 0000000000..f4e7f212c9 --- /dev/null +++ b/apps/indexer/lib/indexer/fetcher/filecoin/beryx_api.ex @@ -0,0 +1,50 @@ +defmodule Indexer.Fetcher.Filecoin.BeryxAPI do + @moduledoc """ + Interacts with the Beryx API to fetch account information based on an Ethereum + address hash + """ + + alias Explorer.Helper + alias HTTPoison.Response + + @doc """ + Fetches account information for a given Ethereum address hash from the Beryx API. + + ## Parameters + - `eth_address_hash` - The Ethereum address hash to fetch information for. + + ## Returns + - `{:ok, map()}`: On success, returns the account information as a map. + - `{:error, integer(), map()}`: On failure, returns the HTTP status code and the error message as a map. + - `{:error, HTTPoison.Error.t()}`: On network or other HTTP errors, returns the error structure. + """ + @spec fetch_account_info(EthereumJSONRPC.address()) :: + {:ok, map()} + | {:error, integer(), map()} + | {:error, HTTPoison.Error.t()} + def fetch_account_info(eth_address_hash) do + config = Application.get_env(:indexer, __MODULE__) + base_url = config |> Keyword.get(:base_url) |> String.trim_trailing("/") + api_token = config[:api_token] + + url = "#{base_url}/mainnet/account/info/#{eth_address_hash}" + + headers = [ + {"Authorization", "Bearer #{api_token}"}, + {"Content-Type", "application/json"} + ] + + case HTTPoison.get(url, headers) do + {:ok, %Response{body: body, status_code: 200}} -> + json = Helper.decode_json(body) + {:ok, json} + + {:ok, %Response{body: body, status_code: status_code}} -> + json = Helper.decode_json(body) + {:error, status_code, json} + + {:error, %HTTPoison.Error{}} = error -> + error + end + end +end diff --git a/apps/indexer/lib/indexer/prometheus/collector/filecoin_pending_address_operations_collector.ex b/apps/indexer/lib/indexer/prometheus/collector/filecoin_pending_address_operations_collector.ex new file mode 100644 index 0000000000..2078412575 --- /dev/null +++ b/apps/indexer/lib/indexer/prometheus/collector/filecoin_pending_address_operations_collector.ex @@ -0,0 +1,29 @@ +defmodule Indexer.Prometheus.Collector.FilecoinPendingAddressOperations do + @moduledoc """ + Custom collector to count number of records in filecoin_pending_address_operations table. + """ + + use Prometheus.Collector + + alias Explorer.Chain.Filecoin.PendingAddressOperation + alias Explorer.Repo + alias Prometheus.Model + + def collect_mf(_registry, callback) do + callback.( + create_gauge( + :filecoin_pending_address_operations, + "Number of records in filecoin_pending_address_operations table", + Repo.aggregate(PendingAddressOperation, :count, timeout: :infinity) + ) + ) + end + + def collect_metrics(:filecoin_pending_address_operations, count) do + Model.gauge_metrics([{count}]) + end + + defp create_gauge(name, help, data) do + Model.create_mf(name, help, :gauge, __MODULE__, data) + end +end diff --git a/apps/indexer/lib/indexer/prometheus/pending_block_operations_collector.ex b/apps/indexer/lib/indexer/prometheus/collector/pending_block_operations_collector.ex similarity index 91% rename from apps/indexer/lib/indexer/prometheus/pending_block_operations_collector.ex rename to apps/indexer/lib/indexer/prometheus/collector/pending_block_operations_collector.ex index e1e836a6bc..56f7f48408 100644 --- a/apps/indexer/lib/indexer/prometheus/pending_block_operations_collector.ex +++ b/apps/indexer/lib/indexer/prometheus/collector/pending_block_operations_collector.ex @@ -1,4 +1,4 @@ -defmodule Indexer.Prometheus.PendingBlockOperationsCollector do +defmodule Indexer.Prometheus.Collector.PendingBlockOperations do @moduledoc """ Custom collector to count number of records in pending_block_operations table. """ diff --git a/apps/indexer/lib/indexer/supervisor.ex b/apps/indexer/lib/indexer/supervisor.ex index 641b4ac654..e56883914c 100644 --- a/apps/indexer/lib/indexer/supervisor.ex +++ b/apps/indexer/lib/indexer/supervisor.ex @@ -195,6 +195,9 @@ defmodule Indexer.Supervisor do configure(Indexer.Fetcher.Celo.EpochBlockOperations.Supervisor, [ [json_rpc_named_arguments: json_rpc_named_arguments, memory_monitor: memory_monitor] ]), + configure(Indexer.Fetcher.Filecoin.AddressInfo.Supervisor, [ + [memory_monitor: memory_monitor] + ]), {Indexer.Fetcher.Beacon.Blob.Supervisor, [[memory_monitor: memory_monitor]]}, # Out-of-band fetchers diff --git a/apps/indexer/test/indexer/block/catchup/fetcher_test.exs b/apps/indexer/test/indexer/block/catchup/fetcher_test.exs index 292b48183c..86fac863c3 100644 --- a/apps/indexer/test/indexer/block/catchup/fetcher_test.exs +++ b/apps/indexer/test/indexer/block/catchup/fetcher_test.exs @@ -52,6 +52,7 @@ defmodule Indexer.Block.Catchup.FetcherTest do InternalTransaction.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) Token.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) TokenBalance.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) + Indexer.Fetcher.Filecoin.AddressInfo.Supervisor.Case.start_supervised!() MissingRangesCollector.start_link([]) MissingRangesManipulator.start_link([]) diff --git a/apps/indexer/test/indexer/block/fetcher_test.exs b/apps/indexer/test/indexer/block/fetcher_test.exs index ce5ff1686a..21416981d0 100644 --- a/apps/indexer/test/indexer/block/fetcher_test.exs +++ b/apps/indexer/test/indexer/block/fetcher_test.exs @@ -275,6 +275,8 @@ defmodule Indexer.Block.FetcherTest do } do block_number = @first_full_block_number + Indexer.Fetcher.Filecoin.AddressInfo.Supervisor.Case.start_supervised!() + if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do case Keyword.fetch!(json_rpc_named_arguments, :variant) do EthereumJSONRPC.Nethermind -> @@ -682,6 +684,8 @@ defmodule Indexer.Block.FetcherTest do } do block_number = 7_374_455 + Indexer.Fetcher.Filecoin.AddressInfo.Supervisor.Case.start_supervised!() + if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do EthereumJSONRPC.Mox |> expect(:json_rpc, 2, fn diff --git a/apps/indexer/test/indexer/block/realtime/fetcher_test.exs b/apps/indexer/test/indexer/block/realtime/fetcher_test.exs index 12e164b634..ed249cf12c 100644 --- a/apps/indexer/test/indexer/block/realtime/fetcher_test.exs +++ b/apps/indexer/test/indexer/block/realtime/fetcher_test.exs @@ -91,6 +91,8 @@ defmodule Indexer.Block.Realtime.FetcherTest do ReplacedTransaction.Supervisor.Case.start_supervised!() + Indexer.Fetcher.Filecoin.AddressInfo.Supervisor.Case.start_supervised!() + # In CELO network, there is a token duality feature where CELO can be used # as both a native chain currency and as an ERC-20 token (GoldToken). # Transactions that transfer CELO are also counted as token transfers, and @@ -596,6 +598,8 @@ defmodule Indexer.Block.Realtime.FetcherTest do block_fetcher: %Indexer.Block.Fetcher{json_rpc_named_arguments: json_rpc_named_arguments} ) + Indexer.Fetcher.Filecoin.AddressInfo.Supervisor.Case.start_supervised!() + ReplacedTransaction.Supervisor.Case.start_supervised!() # In CELO network, there is a token duality feature where CELO can be used diff --git a/apps/indexer/test/support/indexer/fetcher/filecoin_native_address_supervisor_case.ex b/apps/indexer/test/support/indexer/fetcher/filecoin_native_address_supervisor_case.ex new file mode 100644 index 0000000000..6f46db11c5 --- /dev/null +++ b/apps/indexer/test/support/indexer/fetcher/filecoin_native_address_supervisor_case.ex @@ -0,0 +1,17 @@ +defmodule Indexer.Fetcher.Filecoin.AddressInfo.Supervisor.Case do + alias Indexer.Fetcher.Filecoin.AddressInfo + + def start_supervised!(fetcher_arguments \\ []) when is_list(fetcher_arguments) do + merged_fetcher_arguments = + Keyword.merge( + fetcher_arguments, + flush_interval: 50, + max_batch_size: 1, + max_concurrency: 1 + ) + + [merged_fetcher_arguments] + |> AddressInfo.Supervisor.child_spec() + |> ExUnit.Callbacks.start_supervised!() + end +end diff --git a/config/runtime.exs b/config/runtime.exs index 70fdba23d9..7710b3e3c0 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -638,6 +638,14 @@ config :explorer, Explorer.Chain.Metrics, enabled: ConfigHelper.parse_bool_env_var("PUBLIC_METRICS_ENABLED", "false"), update_period_hours: ConfigHelper.parse_integer_env_var("PUBLIC_METRICS_UPDATE_PERIOD_HOURS", 24) +config :explorer, Explorer.Chain.Filecoin.NativeAddress, + network_prefix: ConfigHelper.parse_catalog_value("FILECOIN_NETWORK_PREFIX", ["f", "t"], true, "f") + +config :explorer, Explorer.Migrator.FilecoinPendingAddressOperations, + enabled: ConfigHelper.chain_type() == :filecoin, + batch_size: ConfigHelper.parse_integer_env_var("FILECOIN_PENDING_ADDRESS_OPERATIONS_MIGRATION_BATCH_SIZE", 100), + concurrency: ConfigHelper.parse_integer_env_var("FILECOIN_PENDING_ADDRESS_OPERATIONS_MIGRATION_CONCURRENCY", 1) + ############### ### Indexer ### ############### @@ -1052,6 +1060,21 @@ config :indexer, Indexer.Fetcher.Celo.EpochBlockOperations.Supervisor, enabled: celo_epoch_fetchers_enabled?, disabled?: not celo_epoch_fetchers_enabled? +config :indexer, Indexer.Fetcher.Filecoin.BeryxAPI, + base_url: ConfigHelper.safe_get_env("BERYX_API_BASE_URL", "https://api.zondax.ch/fil/data/v3"), + api_token: System.get_env("BERYX_API_TOKEN") + +filecoin_native_address_fetcher_enabled? = + ConfigHelper.chain_type() == :filecoin and + not ConfigHelper.parse_bool_env_var("INDEXER_DISABLE_FILECOIN_ADDRESS_INFO_FETCHER") + +config :indexer, Indexer.Fetcher.Filecoin.AddressInfo.Supervisor, + enabled: filecoin_native_address_fetcher_enabled?, + disabled?: not filecoin_native_address_fetcher_enabled? + +config :indexer, Indexer.Fetcher.Filecoin.AddressInfo, + concurrency: ConfigHelper.parse_integer_env_var("INDEXER_FILECOIN_ADDRESS_INFO_CONCURRENCY", 1) + Code.require_file("#{config_env()}.exs", "config/runtime") for config <- "../apps/*/config/runtime/#{config_env()}.exs" |> Path.expand(__DIR__) |> Path.wildcard() do diff --git a/cspell.json b/cspell.json index 6d3d91b6b3..3fbe051f38 100644 --- a/cspell.json +++ b/cspell.json @@ -47,6 +47,7 @@ "bafybeihxuj", "balancemulti", "benchee", + "beryx", "besu", "bignumber", "bigserial", @@ -133,6 +134,7 @@ "Cyclomatic", "cypherpunk", "czilladx", + "datacap", "datapoint", "datepicker", "DATETIME", @@ -178,6 +180,7 @@ "errorb", "erts", "Ethash", + "ethaccount", "etherchain", "ethprice", "ethsupply", @@ -378,6 +381,7 @@ "outcoming", "overengineering", "pawesome", + "paych", "pbcopy", "peeker", "peekers", @@ -587,6 +591,7 @@ "valuemin", "valuenow", "varint", + "verifreg", "verifyproxycontract", "verifysourcecode", "viewerjs", diff --git a/docker-compose/envs/common-blockscout.env b/docker-compose/envs/common-blockscout.env index 5ec959df46..cf21ce1154 100644 --- a/docker-compose/envs/common-blockscout.env +++ b/docker-compose/envs/common-blockscout.env @@ -266,6 +266,13 @@ INDEXER_DISABLE_INTERNAL_TRANSACTIONS_FETCHER=false # INDEXER_CELO_VALIDATOR_GROUP_VOTES_BATCH_SIZE=200000 # INDEXER_DISABLE_CELO_EPOCH_FETCHER=false # INDEXER_DISABLE_CELO_VALIDATOR_GROUP_VOTES_FETCHER=false +# BERYX_API_TOKEN= +# BERYX_API_BASE_URL= +# FILECOIN_NETWORK_PREFIX=f +# FILECOIN_PENDING_ADDRESS_OPERATIONS_MIGRATION_BATCH_SIZE= +# FILECOIN_PENDING_ADDRESS_OPERATIONS_MIGRATION_CONCURRENCY= +# INDEXER_DISABLE_FILECOIN_ADDRESS_INFO_FETCHER=false +# INDEXER_FILECOIN_ADDRESS_INFO_CONCURRENCY=1 # INDEXER_ARBITRUM_MISSED_MESSAGES_BLOCKS_DEPTH= # INDEXER_REALTIME_FETCHER_MAX_GAP= # INDEXER_FETCHER_INIT_QUERY_LIMIT= diff --git a/mix.lock b/mix.lock index 6df0f6f0aa..03cf8a5e1f 100644 --- a/mix.lock +++ b/mix.lock @@ -8,6 +8,7 @@ "bcrypt_elixir": {:hex, :bcrypt_elixir, "3.1.0", "0b110a9a6c619b19a7f73fa3004aa11d6e719a67e672d1633dc36b6b2290a0f7", [:make, :mix], [{:comeonin, "~> 5.3", [hex: :comeonin, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "2ad2acb5a8bc049e8d5aa267802631912bb80d5f4110a178ae7999e69dca1bf7"}, "benchee": {:hex, :benchee, "1.3.1", "c786e6a76321121a44229dde3988fc772bca73ea75170a73fd5f4ddf1af95ccf", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}, {:statistex, "~> 1.0", [hex: :statistex, repo: "hexpm", optional: false]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "76224c58ea1d0391c8309a8ecbfe27d71062878f59bd41a390266bf4ac1cc56d"}, "benchee_csv": {:hex, :benchee_csv, "1.0.0", "0b3b9223290bfcb8003552705bec9bcf1a89b4a83b70bd686e45295c264f3d16", [:mix], [{:benchee, ">= 0.99.0 and < 2.0.0", [hex: :benchee, repo: "hexpm", optional: false]}, {:csv, "~> 2.0", [hex: :csv, repo: "hexpm", optional: false]}], "hexpm", "cdefb804c021dcf7a99199492026584be9b5a21d6644ac0d01c81c5d97c520d5"}, + "blake2": {:hex, :blake2, "1.0.4", "8263c69a191142922bc2510f1ffc0de0ae96e8c3bd5e2ad3fac7e87aed94c8b1", [:mix], [], "hexpm", "e9f4120d163ba14d86304195e50745fa18483e6ad2be94c864ae449bbdd6a189"}, "briefly": {:git, "https://github.com/CargoSense/briefly.git", "4836ba322ffb504a102a15cc6e35d928ef97120e", []}, "brotli": {:hex, :brotli, "0.3.2", "59cf45a399098516f1d34f70d8e010e5c9bf326659d3ef34c7cc56793339002b", [:rebar3], [], "hexpm", "9ec3ef9c753f80d0c657b4905193c55e5198f169fa1d1c044d8601d4d931a2ad"}, "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"},