feat: Add Blackfort validators (#10744)

* feat: Add Blackfort validators

* Finishing touches

* Add missing specs, docs, pre-release workflow for Blackfort

* Remove blocks_validated

---------

Co-authored-by: Victor Baranov <baranov.viktor.27@gmail.com>
pull/10739/head
nikitosing 2 months ago committed by GitHub
parent f7a434df17
commit 629620bc52
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
  1. 2
      .github/workflows/config.yml
  2. 97
      .github/workflows/pre-release-blackfort.yml
  3. 51
      .github/workflows/publish-docker-image-for-blackfort.yml
  4. 94
      .github/workflows/release-blackfort.yml
  5. 15
      apps/block_scout_web/lib/block_scout_web/chain.ex
  6. 50
      apps/block_scout_web/lib/block_scout_web/controllers/api/v2/validator_controller.ex
  7. 14
      apps/block_scout_web/lib/block_scout_web/paging_helper.ex
  8. 12
      apps/block_scout_web/lib/block_scout_web/routers/api_router.ex
  9. 21
      apps/block_scout_web/lib/block_scout_web/views/api/v2/validator_view.ex
  10. 5
      apps/explorer/config/config.exs
  11. 2
      apps/explorer/config/dev.exs
  12. 5
      apps/explorer/config/prod.exs
  13. 3
      apps/explorer/config/test.exs
  14. 4
      apps/explorer/lib/explorer/application.ex
  15. 201
      apps/explorer/lib/explorer/chain/blackfort/validator.ex
  16. 97
      apps/explorer/lib/explorer/chain/cache/blackfort_validators_counters.ex
  17. 10
      apps/explorer/lib/explorer/repo.ex
  18. 20
      apps/explorer/priv/blackfort/migrations/20240910112251_add_blackfort_validators.exs
  19. 11
      apps/indexer/lib/indexer/block/realtime/fetcher.ex
  20. 49
      apps/indexer/lib/indexer/fetcher/blackfort/validator.ex
  21. 4
      apps/indexer/lib/indexer/supervisor.ex
  22. 4
      config/config_helper.exs
  23. 2
      config/runtime.exs
  24. 7
      config/runtime/dev.exs
  25. 6
      config/runtime/prod.exs
  26. 5
      cspell.json

@ -49,7 +49,7 @@ jobs:
// Add/remove CI matrix chain types here
const defaultChainTypes = ["default"];
const chainTypes = ["ethereum", "polygon_zkevm", "rsk", "stability", "filecoin", "optimism", "arbitrum", "celo", "zetachain", "zksync", "shibarium"];
const chainTypes = ["ethereum", "polygon_zkevm", "rsk", "stability", "filecoin", "optimism", "arbitrum", "celo", "zetachain", "zksync", "shibarium", "blackfort"];
const extraChainTypes = ["suave", "polygon_edge"];
// Chain type matrix we use in master branch

@ -0,0 +1,97 @@
name: Pre-release for Blackfort
on:
workflow_dispatch:
inputs:
number:
type: number
required: true
env:
OTP_VERSION: ${{ vars.OTP_VERSION }}
ELIXIR_VERSION: ${{ vars.ELIXIR_VERSION }}
jobs:
push_to_registry:
name: Push Docker image to Docker Hub
runs-on: ubuntu-latest
env:
RELEASE_VERSION: ${{ vars.RELEASE_VERSION }}
steps:
- uses: actions/checkout@v4
- name: Setup repo
uses: ./.github/actions/setup-repo
id: setup
with:
docker-username: ${{ secrets.DOCKER_USERNAME }}
docker-password: ${{ secrets.DOCKER_PASSWORD }}
docker-remote-multi-platform: true
docker-arm-host: ${{ secrets.ARM_RUNNER_HOSTNAME }}
docker-arm-host-key: ${{ secrets.ARM_RUNNER_KEY }}
- name: Build and push Docker image for Blackfort (indexer + API)
uses: docker/build-push-action@v5
with:
context: .
file: ./docker/Dockerfile
push: true
tags: blockscout/blockscout-blackfort:${{ env.RELEASE_VERSION }}-alpha.${{ inputs.number }}
labels: ${{ steps.setup.outputs.docker-labels }}
platforms: |
linux/amd64
linux/arm64/v8
build-args: |
DISABLE_WEBAPP=false
API_V1_READ_METHODS_DISABLED=false
API_V1_WRITE_METHODS_DISABLED=false
CACHE_EXCHANGE_RATES_PERIOD=
CACHE_TOTAL_GAS_USAGE_COUNTER_ENABLED=
CACHE_ADDRESS_WITH_BALANCES_UPDATE_INTERVAL=
ADMIN_PANEL_ENABLED=false
BLOCKSCOUT_VERSION=v${{ env.RELEASE_VERSION }}-alpha.${{ inputs.number }}
RELEASE_VERSION=${{ env.RELEASE_VERSION }}
CHAIN_TYPE=blackfort
- name: Build and push Docker image for Blackfort (indexer)
uses: docker/build-push-action@v5
with:
context: .
file: ./docker/Dockerfile
push: true
tags: blockscout/blockscout-blackfort:${{ env.RELEASE_VERSION }}-alpha.${{ inputs.number }}-indexer
labels: ${{ steps.setup.outputs.docker-labels }}
platforms: |
linux/amd64
linux/arm64/v8
build-args: |
DISABLE_API=true
DISABLE_WEBAPP=true
CACHE_EXCHANGE_RATES_PERIOD=
CACHE_TOTAL_GAS_USAGE_COUNTER_ENABLED=
CACHE_ADDRESS_WITH_BALANCES_UPDATE_INTERVAL=
ADMIN_PANEL_ENABLED=false
BLOCKSCOUT_VERSION=v${{ env.RELEASE_VERSION }}-alpha.${{ inputs.number }}
RELEASE_VERSION=${{ env.RELEASE_VERSION }}
CHAIN_TYPE=blackfort
- name: Build and push Docker image for Blackfort (API)
uses: docker/build-push-action@v5
with:
context: .
file: ./docker/Dockerfile
push: true
tags: blockscout/blockscout-blackfort:${{ env.RELEASE_VERSION }}-alpha.${{ inputs.number }}-api
labels: ${{ steps.setup.outputs.docker-labels }}
platforms: |
linux/amd64
linux/arm64/v8
build-args: |
DISABLE_INDEXER=true
DISABLE_WEBAPP=true
CACHE_EXCHANGE_RATES_PERIOD=
CACHE_TOTAL_GAS_USAGE_COUNTER_ENABLED=
CACHE_ADDRESS_WITH_BALANCES_UPDATE_INTERVAL=
ADMIN_PANEL_ENABLED=false
BLOCKSCOUT_VERSION=v${{ env.RELEASE_VERSION }}-alpha.${{ inputs.number }}
RELEASE_VERSION=${{ env.RELEASE_VERSION }}
CHAIN_TYPE=blackfort

@ -0,0 +1,51 @@
name: Blackfort Publish Docker image
on:
workflow_dispatch:
push:
branches:
- production-blackfort
env:
OTP_VERSION: ${{ vars.OTP_VERSION }}
ELIXIR_VERSION: ${{ vars.ELIXIR_VERSION }}
jobs:
push_to_registry:
name: Push Docker image to Docker Hub
runs-on: ubuntu-latest
env:
RELEASE_VERSION: ${{ vars.RELEASE_VERSION }}
DOCKER_CHAIN_NAME: blackfort
steps:
- uses: actions/checkout@v4
- name: Setup repo
uses: ./.github/actions/setup-repo
id: setup
with:
docker-username: ${{ secrets.DOCKER_USERNAME }}
docker-password: ${{ secrets.DOCKER_PASSWORD }}
docker-remote-multi-platform: true
docker-arm-host: ${{ secrets.ARM_RUNNER_HOSTNAME }}
docker-arm-host-key: ${{ secrets.ARM_RUNNER_KEY }}
- name: Build and push Docker image
uses: docker/build-push-action@v5
with:
context: .
file: ./docker/Dockerfile
push: true
tags: blockscout/blockscout-${{ env.DOCKER_CHAIN_NAME }}:${{ env.RELEASE_VERSION }}-postrelease-${{ env.SHORT_SHA }}
labels: ${{ steps.setup.outputs.docker-labels }}
platforms: |
linux/amd64
linux/arm64/v8
build-args: |
CACHE_EXCHANGE_RATES_PERIOD=
API_V1_READ_METHODS_DISABLED=false
DISABLE_WEBAPP=false
API_V1_WRITE_METHODS_DISABLED=false
CACHE_TOTAL_GAS_USAGE_COUNTER_ENABLED=
ADMIN_PANEL_ENABLED=false
CACHE_ADDRESS_WITH_BALANCES_UPDATE_INTERVAL=
BLOCKSCOUT_VERSION=v${{ env.RELEASE_VERSION }}-beta.+commit.${{ env.SHORT_SHA }}
RELEASE_VERSION=${{ env.RELEASE_VERSION }}
CHAIN_TYPE=blackfort

@ -0,0 +1,94 @@
name: Release for Blackfort
on:
release:
types: [published]
env:
OTP_VERSION: ${{ vars.OTP_VERSION }}
ELIXIR_VERSION: ${{ vars.ELIXIR_VERSION }}
jobs:
push_to_registry:
name: Push Docker image to Docker Hub
runs-on: ubuntu-latest
env:
RELEASE_VERSION: ${{ vars.RELEASE_VERSION }}
steps:
- uses: actions/checkout@v4
- name: Setup repo
uses: ./.github/actions/setup-repo
id: setup
with:
docker-username: ${{ secrets.DOCKER_USERNAME }}
docker-password: ${{ secrets.DOCKER_PASSWORD }}
docker-remote-multi-platform: true
docker-arm-host: ${{ secrets.ARM_RUNNER_HOSTNAME }}
docker-arm-host-key: ${{ secrets.ARM_RUNNER_KEY }}
- name: Build and push Docker image for Blackfort (indexer + API)
uses: docker/build-push-action@v5
with:
context: .
file: ./docker/Dockerfile
push: true
tags: blockscout/blockscout-blackfort:latest, blockscout/blockscout-blackfort:${{ env.RELEASE_VERSION }}
labels: ${{ steps.setup.outputs.docker-labels }}
platforms: |
linux/amd64
linux/arm64/v8
build-args: |
DISABLE_WEBAPP=false
API_V1_READ_METHODS_DISABLED=false
API_V1_WRITE_METHODS_DISABLED=false
CACHE_EXCHANGE_RATES_PERIOD=
CACHE_TOTAL_GAS_USAGE_COUNTER_ENABLED=
CACHE_ADDRESS_WITH_BALANCES_UPDATE_INTERVAL=
ADMIN_PANEL_ENABLED=false
BLOCKSCOUT_VERSION=v${{ env.RELEASE_VERSION }}-beta
RELEASE_VERSION=${{ env.RELEASE_VERSION }}
CHAIN_TYPE=blackfort
- name: Build and push Docker image for Blackfort (indexer)
uses: docker/build-push-action@v5
with:
context: .
file: ./docker/Dockerfile
push: true
tags: blockscout/blockscout-blackfort:${{ env.RELEASE_VERSION }}-indexer
labels: ${{ steps.setup.outputs.docker-labels }}
platforms: |
linux/amd64
linux/arm64/v8
build-args: |
DISABLE_API=true
DISABLE_WEBAPP=true
CACHE_EXCHANGE_RATES_PERIOD=
CACHE_TOTAL_GAS_USAGE_COUNTER_ENABLED=
CACHE_ADDRESS_WITH_BALANCES_UPDATE_INTERVAL=
ADMIN_PANEL_ENABLED=false
BLOCKSCOUT_VERSION=v${{ env.RELEASE_VERSION }}-beta
RELEASE_VERSION=${{ env.RELEASE_VERSION }}
CHAIN_TYPE=blackfort
- name: Build and push Docker image for Blackfort (API)
uses: docker/build-push-action@v5
with:
context: .
file: ./docker/Dockerfile
push: true
tags: blockscout/blockscout-blackfort:${{ env.RELEASE_VERSION }}-api
labels: ${{ steps.setup.outputs.docker-labels }}
platforms: |
linux/amd64
linux/arm64/v8
build-args: |
DISABLE_INDEXER=true
DISABLE_WEBAPP=true
CACHE_EXCHANGE_RATES_PERIOD=
CACHE_TOTAL_GAS_USAGE_COUNTER_ENABLED=
CACHE_ADDRESS_WITH_BALANCES_UPDATE_INTERVAL=
ADMIN_PANEL_ENABLED=false
BLOCKSCOUT_VERSION=v${{ env.RELEASE_VERSION }}-beta
RELEASE_VERSION=${{ env.RELEASE_VERSION }}
CHAIN_TYPE=blackfort

@ -508,6 +508,21 @@ defmodule BlockScoutWeb.Chain do
[paging_options: %{@default_paging_options | key: %{block_index: index}}]
end
# Clause for `Explorer.Chain.Blackfort.Validator`,
# returned by `BlockScoutWeb.API.V2.ValidatorController.blackfort_validators_list/2` (`/api/v2/validators/blackfort`)
def paging_options(%{
"address_hash" => address_hash_string
}) do
[
paging_options: %{
@default_paging_options
| key: %{
address_hash: parse_address_hash(address_hash_string)
}
}
]
end
def paging_options(_params), do: [paging_options: @default_paging_options]
def put_key_value_to_paging_options([paging_options: paging_options], key, value) do

@ -1,13 +1,15 @@
defmodule BlockScoutWeb.API.V2.ValidatorController do
use BlockScoutWeb, :controller
alias Explorer.Chain.Cache.StabilityValidatorsCounters
alias Explorer.Chain.Blackfort.Validator, as: ValidatorBlackfort
alias Explorer.Chain.Cache.{BlackfortValidatorsCounters, StabilityValidatorsCounters}
alias Explorer.Chain.Stability.Validator, as: ValidatorStability
import BlockScoutWeb.PagingHelper,
only: [
delete_parameters_from_next_page_params: 1,
stability_validators_state_options: 1,
validators_blackfort_sorting: 1,
validators_stability_sorting: 1
]
@ -71,6 +73,52 @@ defmodule BlockScoutWeb.API.V2.ValidatorController do
})
end
@doc """
Function to handle GET requests to `/api/v2/validators/blackfort` endpoint.
"""
@spec blackfort_validators_list(Plug.Conn.t(), map()) :: Plug.Conn.t()
def blackfort_validators_list(conn, params) do
options =
[
necessity_by_association: %{
[address: [:names, :smart_contract, :proxy_implementations]] => :optional
}
]
|> Keyword.merge(@api_true)
|> Keyword.merge(paging_options(params))
|> Keyword.merge(validators_blackfort_sorting(params))
{validators, next_page} = options |> ValidatorBlackfort.get_paginated_validators() |> split_list_by_page()
next_page_params =
next_page
|> next_page_params(
validators,
delete_parameters_from_next_page_params(params),
&ValidatorBlackfort.next_page_params/1
)
conn
|> render(:blackfort_validators, %{validators: validators, next_page_params: next_page_params})
end
@doc """
Function to handle GET requests to `/api/v2/validators/blackfort/counters` endpoint.
"""
@spec blackfort_validators_counters(Plug.Conn.t(), map()) :: Plug.Conn.t()
def blackfort_validators_counters(conn, _params) do
%{
validators_counter: validators_counter,
new_validators_counter: new_validators_counter
} = BlackfortValidatorsCounters.get_counters(@api_true)
conn
|> json(%{
validators_counter: validators_counter,
new_validators_counter_24h: new_validators_counter
})
end
defp calculate_active_validators_percentage(active_validators_counter, validators_counter) do
if Decimal.compare(validators_counter, Decimal.new(0)) == :gt do
active_validators_counter

@ -334,4 +334,18 @@ defmodule BlockScoutWeb.PagingHelper do
defp do_mud_records_sorting("key1", "asc"), do: [asc_nulls_first: :key1]
defp do_mud_records_sorting("key1", "desc"), do: [desc_nulls_last: :key1]
defp do_mud_records_sorting(_, _), do: []
@spec validators_blackfort_sorting(%{required(String.t()) => String.t()}) :: [
{:sorting, SortingHelper.sorting_params()}
]
def validators_blackfort_sorting(%{"sort" => sort_field, "order" => order}) do
[sorting: do_validators_blackfort_sorting(sort_field, order)]
end
def validators_blackfort_sorting(_), do: []
defp do_validators_blackfort_sorting("address_hash", "asc"), do: [asc_nulls_first: :address_hash]
defp do_validators_blackfort_sorting("address_hash", "desc"), do: [desc_nulls_last: :address_hash]
defp do_validators_blackfort_sorting(_, _), do: []
end

@ -327,11 +327,21 @@ defmodule BlockScoutWeb.Routers.ApiRouter do
end
scope "/validators" do
if Application.compile_env(:explorer, :chain_type) == :stability do
case Application.compile_env(:explorer, :chain_type) do
:stability ->
scope "/stability" do
get("/", V2.ValidatorController, :stability_validators_list)
get("/counters", V2.ValidatorController, :stability_validators_counters)
end
:blackfort ->
scope "/blackfort" do
get("/", V2.ValidatorController, :blackfort_validators_list)
get("/counters", V2.ValidatorController, :blackfort_validators_counters)
end
_ ->
nil
end
end

@ -4,14 +4,31 @@ defmodule BlockScoutWeb.API.V2.ValidatorView do
alias BlockScoutWeb.API.V2.Helper
def render("stability_validators.json", %{validators: validators, next_page_params: next_page_params}) do
%{"items" => Enum.map(validators, &prepare_validator(&1)), "next_page_params" => next_page_params}
%{"items" => Enum.map(validators, &prepare_stability_validator(&1)), "next_page_params" => next_page_params}
end
defp prepare_validator(validator) do
def render("blackfort_validators.json", %{validators: validators, next_page_params: next_page_params}) do
%{"items" => Enum.map(validators, &prepare_blackfort_validator(&1)), "next_page_params" => next_page_params}
end
defp prepare_stability_validator(validator) do
%{
"address" => Helper.address_with_info(nil, validator.address, validator.address_hash, true),
"state" => validator.state,
"blocks_validated_count" => validator.blocks_validated
}
end
defp prepare_blackfort_validator(validator) do
%{
"address" => Helper.address_with_info(nil, validator.address, validator.address_hash, true),
"name" => validator.name,
"commission" => validator.commission,
"self_bonded_amount" => validator.self_bonded_amount,
"delegated_amount" => validator.delegated_amount,
"slashing_status_is_slashed" => validator.slashing_status_is_slashed,
"slashing_status_by_block" => validator.slashing_status_by_block,
"slashing_status_multiplier" => validator.slashing_status_multiplier
}
end
end

@ -81,6 +81,11 @@ config :explorer, Explorer.Chain.Cache.StabilityValidatorsCounters,
enable_consolidation: true,
update_interval_in_milliseconds: update_interval_in_milliseconds_default
config :explorer, Explorer.Chain.Cache.BlackfortValidatorsCounters,
enabled: true,
enable_consolidation: true,
update_interval_in_milliseconds: update_interval_in_milliseconds_default
config :explorer, Explorer.Chain.Cache.TransactionActionTokensData, enabled: true
config :explorer, Explorer.Chain.Cache.TransactionActionUniswapPools, enabled: true

@ -46,6 +46,8 @@ config :explorer, Explorer.Repo.Mud, timeout: :timer.seconds(80)
config :explorer, Explorer.Repo.ShrunkInternalTransactions, timeout: :timer.seconds(80)
config :explorer, Explorer.Repo.Blackfort, timeout: :timer.seconds(80)
config :explorer, Explorer.Tracer, env: "dev", disabled?: true
config :logger, :explorer,

@ -94,6 +94,11 @@ config :explorer, Explorer.Repo.ShrunkInternalTransactions,
timeout: :timer.seconds(60),
ssl_opts: [verify: :verify_none]
config :explorer, Explorer.Repo.Blackfort,
prepare: :unnamed,
timeout: :timer.seconds(60),
ssl_opts: [verify: :verify_none]
config :explorer, Explorer.Tracer, env: "production", disabled?: true
config :logger, :explorer,

@ -68,7 +68,8 @@ for repo <- [
Explorer.Repo.Filecoin,
Explorer.Repo.Stability,
Explorer.Repo.Mud,
Explorer.Repo.ShrunkInternalTransactions
Explorer.Repo.ShrunkInternalTransactions,
Explorer.Repo.Blackfort
] do
config :explorer, repo,
database: database,

@ -144,6 +144,7 @@ defmodule Explorer.Application do
configure(Explorer.Migrator.RestoreOmittedWETHTransfers),
configure(Explorer.Migrator.FilecoinPendingAddressOperations),
configure_mode_dependent_process(Explorer.Migrator.ShrinkInternalTransactions, :indexer),
configure_chain_type_dependent_process(Explorer.Chain.Cache.BlackfortValidatorsCounters, :blackfort),
configure_chain_type_dependent_process(Explorer.Chain.Cache.StabilityValidatorsCounters, :stability),
configure_mode_dependent_process(Explorer.Migrator.SanitizeMissingTokenBalances, :indexer)
]
@ -168,7 +169,8 @@ defmodule Explorer.Application do
Explorer.Repo.BridgedTokens,
Explorer.Repo.Filecoin,
Explorer.Repo.Stability,
Explorer.Repo.ShrunkInternalTransactions
Explorer.Repo.ShrunkInternalTransactions,
Explorer.Repo.Blackfort
]
else
[]

@ -0,0 +1,201 @@
defmodule Explorer.Chain.Blackfort.Validator do
@moduledoc """
Blackfort validators
"""
use Explorer.Schema
alias Explorer.Chain.{Address, Import}
alias Explorer.Chain.Hash.Address, as: HashAddress
alias Explorer.{Chain, Repo, SortingHelper}
require Logger
@default_sorting [
asc: :address_hash
]
@primary_key false
typed_schema "validators_blackfort" do
field(:address_hash, HashAddress, primary_key: true)
field(:name, :binary)
field(:commission, :integer)
field(:self_bonded_amount, :decimal)
field(:delegated_amount, :decimal)
field(:slashing_status_is_slashed, :boolean, default: false)
field(:slashing_status_by_block, :integer)
field(:slashing_status_multiplier, :integer)
has_one(:address, Address, foreign_key: :hash, references: :address_hash)
timestamps()
end
@required_attrs ~w(address_hash)a
@optional_attrs ~w(name commission self_bonded_amount delegated_amount)a
def changeset(%__MODULE__{} = validator, attrs) do
validator
|> cast(attrs, @required_attrs ++ @optional_attrs)
|> validate_required(@required_attrs)
|> unique_constraint(:address_hash)
end
@doc """
Get validators list.
Keyword could contain:
- paging_options
- necessity_by_association
- sorting (supported by `Explorer.SortingHelper` module)
- state (one of `@state_enum`)
"""
@spec get_paginated_validators(keyword()) :: [t()]
def get_paginated_validators(options \\ []) do
paging_options = Keyword.get(options, :paging_options, Chain.default_paging_options())
necessity_by_association = Keyword.get(options, :necessity_by_association, %{})
sorting = Keyword.get(options, :sorting, [])
__MODULE__
|> Chain.join_associations(necessity_by_association)
|> SortingHelper.apply_sorting(sorting, @default_sorting)
|> SortingHelper.page_with_sorting(paging_options, sorting, @default_sorting)
|> Chain.select_repo(options).all()
end
@doc """
Get all validators
"""
@spec get_all_validators(keyword()) :: [t()]
def get_all_validators(options \\ []) do
__MODULE__
|> Chain.select_repo(options).all()
end
@doc """
Delete validators by address hashes
"""
@spec delete_validators_by_address_hashes([binary() | HashAddress.t()]) :: {non_neg_integer(), nil | []} | :ignore
def delete_validators_by_address_hashes(list) when is_list(list) and length(list) > 0 do
__MODULE__
|> where([vs], vs.address_hash in ^list)
|> Repo.delete_all()
end
def delete_validators_by_address_hashes(_), do: :ignore
@doc """
Insert validators
"""
@spec insert_validators([map()]) :: {non_neg_integer(), nil | []}
def insert_validators(validators) do
Repo.insert_all(__MODULE__, validators,
on_conflict: {:replace_all_except, [:inserted_at]},
conflict_target: [:address_hash]
)
end
@doc """
Append timestamps (:inserted_at, :updated_at)
"""
@spec append_timestamps(map()) :: map()
def append_timestamps(validator) do
Map.merge(validator, Import.timestamps())
end
@doc """
Derive next page params from %Explorer.Chain.Blackfort.Validator{}
"""
@spec next_page_params(t()) :: map()
def next_page_params(%__MODULE__{address_hash: address_hash}) do
%{"address_hash" => address_hash}
end
@doc """
Returns dynamic query for validated blocks count. Needed for SortingHelper
"""
@spec dynamic_validated_blocks() :: Ecto.Query.dynamic_expr()
def dynamic_validated_blocks do
dynamic(
[vs],
fragment(
"SELECT count(*) FROM blocks WHERE miner_hash = ?",
vs.address_hash
)
)
end
@doc """
Returns total count of validators.
"""
@spec count_validators() :: integer()
def count_validators do
Repo.aggregate(__MODULE__, :count, :address_hash)
end
@doc """
Returns count of new validators (inserted withing last 24h).
"""
@spec count_new_validators() :: integer()
def count_new_validators do
__MODULE__
|> where([vs], vs.inserted_at >= ago(1, "day"))
|> Repo.aggregate(:count, :address_hash)
end
@doc """
Fetch list of Blackfort validators
"""
@spec fetch_validators_list() :: {:ok, list()} | :error
def fetch_validators_list do
case HTTPoison.get(validator_url(), [], follow_redirect: true) do
{:ok, %HTTPoison.Response{status_code: 200, body: body}} ->
body |> Jason.decode() |> parse_validators_info()
error ->
Logger.error("Failed to fetch blackfort validator info: #{inspect(error)}")
:error
end
end
defp parse_validators_info({:ok, validators}) do
{:ok,
validators
|> Enum.map(fn %{
"address" => address_hash_string,
"name" => name,
"commission" => commission,
"self_bonded_amount" => self_bonded_amount,
"delegated_amount" => delegated_amount,
"slashing_status" => %{
"is_slashed" => slashing_status_is_slashed,
"by_block" => slashing_status_by_block,
"multiplier" => slashing_status_multiplier
}
} ->
{:ok, address_hash} = HashAddress.cast(address_hash_string)
%{
address_hash: address_hash,
name: name,
commission: parse_number(commission),
self_bonded_amount: parse_number(self_bonded_amount),
delegated_amount: parse_number(delegated_amount),
slashing_status_is_slashed: slashing_status_is_slashed,
slashing_status_by_block: slashing_status_by_block,
slashing_status_multiplier: slashing_status_multiplier
}
end)}
end
defp parse_validators_info({:error, error}) do
Logger.error("Failed to parse blackfort validator info: #{inspect(error)}")
:error
end
defp validator_url do
Application.get_env(:explorer, __MODULE__)[:api_url]
end
defp parse_number(string) do
{number, _} = Integer.parse(string)
number
end
end

@ -0,0 +1,97 @@
defmodule Explorer.Chain.Cache.BlackfortValidatorsCounters do
@moduledoc """
Counts and store counters of validators blackfort.
It loads the count asynchronously and in a time interval of 30 minutes.
"""
use GenServer
alias Explorer.Chain
alias Explorer.Chain.Blackfort.Validator, as: ValidatorBlackfort
@validators_counter_key "blackfort_validators_counter"
@new_validators_counter_key "new_blackfort_validators_counter"
# It is undesirable to automatically start the consolidation in all environments.
# Consider the test environment: if the consolidation initiates but does not
# finish before a test ends, that test will fail. This way, hundreds of
# tests were failing before disabling the consolidation and the scheduler in
# the test env.
config = Application.compile_env(:explorer, __MODULE__)
@enable_consolidation Keyword.get(config, :enable_consolidation)
@update_interval_in_milliseconds Keyword.get(config, :update_interval_in_milliseconds)
@doc """
Starts a process to periodically update validators blackfort counters
"""
@spec start_link(term()) :: GenServer.on_start()
def start_link(_) do
GenServer.start_link(__MODULE__, [], name: __MODULE__)
end
@impl true
def init(_args) do
{:ok, %{consolidate?: @enable_consolidation}, {:continue, :ok}}
end
defp schedule_next_consolidation do
Process.send_after(self(), :consolidate, @update_interval_in_milliseconds)
end
@impl true
def handle_continue(:ok, %{consolidate?: true} = state) do
consolidate()
schedule_next_consolidation()
{:noreply, state}
end
@impl true
def handle_continue(:ok, state) do
{:noreply, state}
end
@impl true
def handle_info(:consolidate, state) do
consolidate()
schedule_next_consolidation()
{:noreply, state}
end
@doc """
Fetches values for a blackfort validators counters from the `last_fetched_counters` table.
"""
@spec get_counters(Keyword.t()) :: map()
def get_counters(options) do
%{
validators_counter: Chain.get_last_fetched_counter(@validators_counter_key, options),
new_validators_counter: Chain.get_last_fetched_counter(@new_validators_counter_key, options)
}
end
@doc """
Consolidates the info by populating the `last_fetched_counters` table with the current database information.
"""
@spec consolidate() :: any()
def consolidate do
tasks = [
Task.async(fn -> ValidatorBlackfort.count_validators() end),
Task.async(fn -> ValidatorBlackfort.count_new_validators() end)
]
[validators_counter, new_validators_counter] = Task.await_many(tasks, :infinity)
Chain.upsert_last_fetched_counter(%{
counter_type: @validators_counter_key,
value: validators_counter
})
Chain.upsert_last_fetched_counter(%{
counter_type: @new_validators_counter_key,
value: new_validators_counter
})
end
end

@ -286,4 +286,14 @@ defmodule Explorer.Repo do
ConfigHelper.init_repo_module(__MODULE__, opts)
end
end
defmodule Blackfort do
use Ecto.Repo,
otp_app: :explorer,
adapter: Ecto.Adapters.Postgres
def init(_, opts) do
ConfigHelper.init_repo_module(__MODULE__, opts)
end
end
end

@ -0,0 +1,20 @@
defmodule Explorer.Repo.Blackfort.Migrations.AddBlackfortValidators do
use Ecto.Migration
def change do
create table(:validators_blackfort, primary_key: false) do
add(:address_hash, :bytea, null: false, primary_key: true)
add(:name, :string)
add(:commission, :smallint)
add(:self_bonded_amount, :numeric, precision: 100)
add(:delegated_amount, :numeric, precision: 100)
add(:slashing_status_is_slashed, :boolean, default: false)
add(:slashing_status_by_block, :bigint)
add(:slashing_status_multiplier, :integer)
timestamps()
end
create_if_not_exists(index(:validators_blackfort, ["address_hash ASC"]))
end
end

@ -170,11 +170,18 @@ defmodule Indexer.Block.Realtime.Fetcher do
Process.cancel_timer(timer)
end
if Application.compile_env(:explorer, :chain_type) == :stability do
case Application.compile_env(:explorer, :chain_type) do
:stability ->
defp fetch_validators_async do
GenServer.cast(Indexer.Fetcher.Stability.Validator, :update_validators_list)
end
else
:blackfort ->
defp fetch_validators_async do
GenServer.cast(Indexer.Fetcher.Blackfort.Validator, :update_validators_list)
end
_ ->
defp fetch_validators_async do
:ignore
end

@ -0,0 +1,49 @@
defmodule Indexer.Fetcher.Blackfort.Validator do
@moduledoc """
GenServer responsible for updating the list of blackfort validators in the database.
"""
use GenServer
alias Explorer.Chain.Blackfort.Validator
def start_link(_) do
GenServer.start_link(__MODULE__, [], name: __MODULE__)
end
@impl true
def init(state) do
GenServer.cast(__MODULE__, :update_validators_list)
{:ok, state}
end
@impl true
def handle_cast(:update_validators_list, state) do
case Validator.fetch_validators_list() do
{:ok, validators} ->
validators_from_db = Validator.get_all_validators()
validators_map =
Enum.reduce(validators, %{}, fn %{address_hash: address_hash}, map ->
Map.put(map, address_hash.bytes, true)
end)
address_hashes_to_drop_from_db =
Enum.flat_map(validators_from_db, fn validator ->
(is_nil(validators_map[validator.address_hash.bytes]) &&
[validator.address_hash]) || []
end)
Validator.delete_validators_by_address_hashes(address_hashes_to_drop_from_db)
validators
|> Enum.map(&Validator.append_timestamps/1)
|> Validator.insert_validators()
_ ->
nil
end
{:noreply, state}
end
end

@ -18,6 +18,7 @@ defmodule Indexer.Supervisor do
alias Indexer.Block.Catchup, as: BlockCatchup
alias Indexer.Block.Realtime, as: BlockRealtime
alias Indexer.Fetcher.Blackfort.Validator, as: ValidatorBlackfort
alias Indexer.Fetcher.CoinBalance.Catchup, as: CoinBalanceCatchup
alias Indexer.Fetcher.CoinBalance.Realtime, as: CoinBalanceRealtime
alias Indexer.Fetcher.Stability.Validator, as: ValidatorStability
@ -285,6 +286,9 @@ defmodule Indexer.Supervisor do
:stability ->
[{ValidatorStability, []} | fetchers]
:blackfort ->
[{ValidatorBlackfort, []} | fetchers]
_ ->
fetchers
end

@ -23,6 +23,7 @@ defmodule ConfigHelper do
:zksync -> base_repos ++ [Explorer.Repo.ZkSync]
:celo -> base_repos ++ [Explorer.Repo.Celo]
:arbitrum -> base_repos ++ [Explorer.Repo.Arbitrum]
:blackfort -> base_repos ++ [Explorer.Repo.Blackfort]
_ -> base_repos
end
@ -316,7 +317,8 @@ defmodule ConfigHelper do
"suave",
"zetachain",
"zksync",
"celo"
"celo",
"blackfort"
]
@spec chain_type() :: atom() | nil

@ -646,6 +646,8 @@ config :explorer, Explorer.Migrator.FilecoinPendingAddressOperations,
batch_size: ConfigHelper.parse_integer_env_var("FILECOIN_PENDING_ADDRESS_OPERATIONS_MIGRATION_BATCH_SIZE", 100),
concurrency: ConfigHelper.parse_integer_env_var("FILECOIN_PENDING_ADDRESS_OPERATIONS_MIGRATION_CONCURRENCY", 1)
config :explorer, Explorer.Chain.Blackfort.Validator, api_url: System.get_env("BLACKFORT_VALIDATOR_API_URL")
###############
### Indexer ###
###############

@ -199,6 +199,13 @@ config :explorer, Explorer.Repo.ShrunkInternalTransactions,
url: System.get_env("DATABASE_URL"),
pool_size: 1
# Configures Blackfort database
config :explorer, Explorer.Repo.Blackfort,
database: database,
hostname: hostname,
url: System.get_env("DATABASE_URL"),
pool_size: 1
variant = Variant.get()
Code.require_file("#{variant}.exs", "apps/explorer/config/dev")

@ -159,6 +159,12 @@ config :explorer, Explorer.Repo.ShrunkInternalTransactions,
pool_size: 1,
ssl: ExplorerConfigHelper.ssl_enabled?()
# Configures Blackfort database
config :explorer, Explorer.Repo.Blackfort,
url: System.get_env("DATABASE_URL"),
pool_size: 1,
ssl: ExplorerConfigHelper.ssl_enabled?()
variant = Variant.get()
Code.require_file("#{variant}.exs", "apps/explorer/config/prod")

@ -18,8 +18,8 @@
"AION",
"AIRTABLE",
"Aiubo",
"alloc",
"alfajores",
"alloc",
"amzootyukbugmx",
"anytrust",
"apikey",
@ -54,6 +54,7 @@
"binwrite",
"bitmask",
"bizbuz",
"Blackfort",
"Blockchair",
"blockheight",
"blockless",
@ -200,9 +201,9 @@
"falala",
"feelin",
"FEVM",
"filecoin",
"Filecoin",
"Filesize",
"filecoin",
"fixidity",
"fkey",
"Floki",

Loading…
Cancel
Save