From 629620bc52a8a88e216ff1f58066fbed49268c36 Mon Sep 17 00:00:00 2001 From: nikitosing <32202610+nikitosing@users.noreply.github.com> Date: Tue, 24 Sep 2024 11:37:35 +0300 Subject: [PATCH] feat: Add Blackfort validators (#10744) * feat: Add Blackfort validators * Finishing touches * Add missing specs, docs, pre-release workflow for Blackfort * Remove blocks_validated --------- Co-authored-by: Victor Baranov --- .github/workflows/config.yml | 2 +- .github/workflows/pre-release-blackfort.yml | 97 +++++++++ .../publish-docker-image-for-blackfort.yml | 51 +++++ .github/workflows/release-blackfort.yml | 94 ++++++++ .../lib/block_scout_web/chain.ex | 15 ++ .../api/v2/validator_controller.ex | 50 ++++- .../lib/block_scout_web/paging_helper.ex | 14 ++ .../lib/block_scout_web/routers/api_router.ex | 20 +- .../views/api/v2/validator_view.ex | 21 +- apps/explorer/config/config.exs | 5 + apps/explorer/config/dev.exs | 2 + apps/explorer/config/prod.exs | 5 + apps/explorer/config/test.exs | 3 +- apps/explorer/lib/explorer/application.ex | 4 +- .../lib/explorer/chain/blackfort/validator.ex | 201 ++++++++++++++++++ .../cache/blackfort_validators_counters.ex | 97 +++++++++ apps/explorer/lib/explorer/repo.ex | 10 + ...0240910112251_add_blackfort_validators.exs | 20 ++ .../lib/indexer/block/realtime/fetcher.ex | 23 +- .../indexer/fetcher/blackfort/validator.ex | 49 +++++ apps/indexer/lib/indexer/supervisor.ex | 4 + config/config_helper.exs | 4 +- config/runtime.exs | 2 + config/runtime/dev.exs | 7 + config/runtime/prod.exs | 6 + cspell.json | 5 +- 26 files changed, 789 insertions(+), 22 deletions(-) create mode 100644 .github/workflows/pre-release-blackfort.yml create mode 100644 .github/workflows/publish-docker-image-for-blackfort.yml create mode 100644 .github/workflows/release-blackfort.yml create mode 100644 apps/explorer/lib/explorer/chain/blackfort/validator.ex create mode 100644 apps/explorer/lib/explorer/chain/cache/blackfort_validators_counters.ex create mode 100644 apps/explorer/priv/blackfort/migrations/20240910112251_add_blackfort_validators.exs create mode 100644 apps/indexer/lib/indexer/fetcher/blackfort/validator.ex diff --git a/.github/workflows/config.yml b/.github/workflows/config.yml index 33237b7202..17cb27192c 100644 --- a/.github/workflows/config.yml +++ b/.github/workflows/config.yml @@ -49,7 +49,7 @@ jobs: // Add/remove CI matrix chain types here const defaultChainTypes = ["default"]; - const chainTypes = ["ethereum", "polygon_zkevm", "rsk", "stability", "filecoin", "optimism", "arbitrum", "celo", "zetachain", "zksync", "shibarium"]; + const chainTypes = ["ethereum", "polygon_zkevm", "rsk", "stability", "filecoin", "optimism", "arbitrum", "celo", "zetachain", "zksync", "shibarium", "blackfort"]; const extraChainTypes = ["suave", "polygon_edge"]; // Chain type matrix we use in master branch diff --git a/.github/workflows/pre-release-blackfort.yml b/.github/workflows/pre-release-blackfort.yml new file mode 100644 index 0000000000..66e730b642 --- /dev/null +++ b/.github/workflows/pre-release-blackfort.yml @@ -0,0 +1,97 @@ +name: Pre-release for Blackfort + +on: + workflow_dispatch: + inputs: + number: + type: number + required: true + +env: + OTP_VERSION: ${{ vars.OTP_VERSION }} + ELIXIR_VERSION: ${{ vars.ELIXIR_VERSION }} + +jobs: + push_to_registry: + name: Push Docker image to Docker Hub + runs-on: ubuntu-latest + env: + RELEASE_VERSION: ${{ vars.RELEASE_VERSION }} + steps: + - uses: actions/checkout@v4 + - name: Setup repo + uses: ./.github/actions/setup-repo + id: setup + with: + docker-username: ${{ secrets.DOCKER_USERNAME }} + docker-password: ${{ secrets.DOCKER_PASSWORD }} + docker-remote-multi-platform: true + docker-arm-host: ${{ secrets.ARM_RUNNER_HOSTNAME }} + docker-arm-host-key: ${{ secrets.ARM_RUNNER_KEY }} + + - name: Build and push Docker image for Blackfort (indexer + API) + uses: docker/build-push-action@v5 + with: + context: . + file: ./docker/Dockerfile + push: true + tags: blockscout/blockscout-blackfort:${{ env.RELEASE_VERSION }}-alpha.${{ inputs.number }} + labels: ${{ steps.setup.outputs.docker-labels }} + platforms: | + linux/amd64 + linux/arm64/v8 + build-args: | + DISABLE_WEBAPP=false + API_V1_READ_METHODS_DISABLED=false + API_V1_WRITE_METHODS_DISABLED=false + CACHE_EXCHANGE_RATES_PERIOD= + CACHE_TOTAL_GAS_USAGE_COUNTER_ENABLED= + CACHE_ADDRESS_WITH_BALANCES_UPDATE_INTERVAL= + ADMIN_PANEL_ENABLED=false + BLOCKSCOUT_VERSION=v${{ env.RELEASE_VERSION }}-alpha.${{ inputs.number }} + RELEASE_VERSION=${{ env.RELEASE_VERSION }} + CHAIN_TYPE=blackfort + + - name: Build and push Docker image for Blackfort (indexer) + uses: docker/build-push-action@v5 + with: + context: . + file: ./docker/Dockerfile + push: true + tags: blockscout/blockscout-blackfort:${{ env.RELEASE_VERSION }}-alpha.${{ inputs.number }}-indexer + labels: ${{ steps.setup.outputs.docker-labels }} + platforms: | + linux/amd64 + linux/arm64/v8 + build-args: | + DISABLE_API=true + DISABLE_WEBAPP=true + CACHE_EXCHANGE_RATES_PERIOD= + CACHE_TOTAL_GAS_USAGE_COUNTER_ENABLED= + CACHE_ADDRESS_WITH_BALANCES_UPDATE_INTERVAL= + ADMIN_PANEL_ENABLED=false + BLOCKSCOUT_VERSION=v${{ env.RELEASE_VERSION }}-alpha.${{ inputs.number }} + RELEASE_VERSION=${{ env.RELEASE_VERSION }} + CHAIN_TYPE=blackfort + + - name: Build and push Docker image for Blackfort (API) + uses: docker/build-push-action@v5 + with: + context: . + file: ./docker/Dockerfile + push: true + tags: blockscout/blockscout-blackfort:${{ env.RELEASE_VERSION }}-alpha.${{ inputs.number }}-api + labels: ${{ steps.setup.outputs.docker-labels }} + platforms: | + linux/amd64 + linux/arm64/v8 + build-args: | + DISABLE_INDEXER=true + DISABLE_WEBAPP=true + CACHE_EXCHANGE_RATES_PERIOD= + CACHE_TOTAL_GAS_USAGE_COUNTER_ENABLED= + CACHE_ADDRESS_WITH_BALANCES_UPDATE_INTERVAL= + ADMIN_PANEL_ENABLED=false + BLOCKSCOUT_VERSION=v${{ env.RELEASE_VERSION }}-alpha.${{ inputs.number }} + RELEASE_VERSION=${{ env.RELEASE_VERSION }} + CHAIN_TYPE=blackfort \ No newline at end of file diff --git a/.github/workflows/publish-docker-image-for-blackfort.yml b/.github/workflows/publish-docker-image-for-blackfort.yml new file mode 100644 index 0000000000..b0fbf41b3b --- /dev/null +++ b/.github/workflows/publish-docker-image-for-blackfort.yml @@ -0,0 +1,51 @@ +name: Blackfort Publish Docker image + +on: + workflow_dispatch: + push: + branches: + - production-blackfort +env: + OTP_VERSION: ${{ vars.OTP_VERSION }} + ELIXIR_VERSION: ${{ vars.ELIXIR_VERSION }} +jobs: + push_to_registry: + name: Push Docker image to Docker Hub + runs-on: ubuntu-latest + env: + RELEASE_VERSION: ${{ vars.RELEASE_VERSION }} + DOCKER_CHAIN_NAME: blackfort + steps: + - uses: actions/checkout@v4 + - name: Setup repo + uses: ./.github/actions/setup-repo + id: setup + with: + docker-username: ${{ secrets.DOCKER_USERNAME }} + docker-password: ${{ secrets.DOCKER_PASSWORD }} + docker-remote-multi-platform: true + docker-arm-host: ${{ secrets.ARM_RUNNER_HOSTNAME }} + docker-arm-host-key: ${{ secrets.ARM_RUNNER_KEY }} + + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: . + file: ./docker/Dockerfile + push: true + tags: blockscout/blockscout-${{ env.DOCKER_CHAIN_NAME }}:${{ env.RELEASE_VERSION }}-postrelease-${{ env.SHORT_SHA }} + labels: ${{ steps.setup.outputs.docker-labels }} + platforms: | + linux/amd64 + linux/arm64/v8 + build-args: | + CACHE_EXCHANGE_RATES_PERIOD= + API_V1_READ_METHODS_DISABLED=false + DISABLE_WEBAPP=false + API_V1_WRITE_METHODS_DISABLED=false + CACHE_TOTAL_GAS_USAGE_COUNTER_ENABLED= + ADMIN_PANEL_ENABLED=false + CACHE_ADDRESS_WITH_BALANCES_UPDATE_INTERVAL= + BLOCKSCOUT_VERSION=v${{ env.RELEASE_VERSION }}-beta.+commit.${{ env.SHORT_SHA }} + RELEASE_VERSION=${{ env.RELEASE_VERSION }} + CHAIN_TYPE=blackfort \ No newline at end of file diff --git a/.github/workflows/release-blackfort.yml b/.github/workflows/release-blackfort.yml new file mode 100644 index 0000000000..76e22b0fde --- /dev/null +++ b/.github/workflows/release-blackfort.yml @@ -0,0 +1,94 @@ +name: Release for Blackfort + +on: + release: + types: [published] + +env: + OTP_VERSION: ${{ vars.OTP_VERSION }} + ELIXIR_VERSION: ${{ vars.ELIXIR_VERSION }} + +jobs: + push_to_registry: + name: Push Docker image to Docker Hub + runs-on: ubuntu-latest + env: + RELEASE_VERSION: ${{ vars.RELEASE_VERSION }} + steps: + - uses: actions/checkout@v4 + - name: Setup repo + uses: ./.github/actions/setup-repo + id: setup + with: + docker-username: ${{ secrets.DOCKER_USERNAME }} + docker-password: ${{ secrets.DOCKER_PASSWORD }} + docker-remote-multi-platform: true + docker-arm-host: ${{ secrets.ARM_RUNNER_HOSTNAME }} + docker-arm-host-key: ${{ secrets.ARM_RUNNER_KEY }} + + - name: Build and push Docker image for Blackfort (indexer + API) + uses: docker/build-push-action@v5 + with: + context: . + file: ./docker/Dockerfile + push: true + tags: blockscout/blockscout-blackfort:latest, blockscout/blockscout-blackfort:${{ env.RELEASE_VERSION }} + labels: ${{ steps.setup.outputs.docker-labels }} + platforms: | + linux/amd64 + linux/arm64/v8 + build-args: | + DISABLE_WEBAPP=false + API_V1_READ_METHODS_DISABLED=false + API_V1_WRITE_METHODS_DISABLED=false + CACHE_EXCHANGE_RATES_PERIOD= + CACHE_TOTAL_GAS_USAGE_COUNTER_ENABLED= + CACHE_ADDRESS_WITH_BALANCES_UPDATE_INTERVAL= + ADMIN_PANEL_ENABLED=false + BLOCKSCOUT_VERSION=v${{ env.RELEASE_VERSION }}-beta + RELEASE_VERSION=${{ env.RELEASE_VERSION }} + CHAIN_TYPE=blackfort + + - name: Build and push Docker image for Blackfort (indexer) + uses: docker/build-push-action@v5 + with: + context: . + file: ./docker/Dockerfile + push: true + tags: blockscout/blockscout-blackfort:${{ env.RELEASE_VERSION }}-indexer + labels: ${{ steps.setup.outputs.docker-labels }} + platforms: | + linux/amd64 + linux/arm64/v8 + build-args: | + DISABLE_API=true + DISABLE_WEBAPP=true + CACHE_EXCHANGE_RATES_PERIOD= + CACHE_TOTAL_GAS_USAGE_COUNTER_ENABLED= + CACHE_ADDRESS_WITH_BALANCES_UPDATE_INTERVAL= + ADMIN_PANEL_ENABLED=false + BLOCKSCOUT_VERSION=v${{ env.RELEASE_VERSION }}-beta + RELEASE_VERSION=${{ env.RELEASE_VERSION }} + CHAIN_TYPE=blackfort + + - name: Build and push Docker image for Blackfort (API) + uses: docker/build-push-action@v5 + with: + context: . + file: ./docker/Dockerfile + push: true + tags: blockscout/blockscout-blackfort:${{ env.RELEASE_VERSION }}-api + labels: ${{ steps.setup.outputs.docker-labels }} + platforms: | + linux/amd64 + linux/arm64/v8 + build-args: | + DISABLE_INDEXER=true + DISABLE_WEBAPP=true + CACHE_EXCHANGE_RATES_PERIOD= + CACHE_TOTAL_GAS_USAGE_COUNTER_ENABLED= + CACHE_ADDRESS_WITH_BALANCES_UPDATE_INTERVAL= + ADMIN_PANEL_ENABLED=false + BLOCKSCOUT_VERSION=v${{ env.RELEASE_VERSION }}-beta + RELEASE_VERSION=${{ env.RELEASE_VERSION }} + CHAIN_TYPE=blackfort \ No newline at end of file diff --git a/apps/block_scout_web/lib/block_scout_web/chain.ex b/apps/block_scout_web/lib/block_scout_web/chain.ex index b6b4ddd1c3..87bd279ae8 100644 --- a/apps/block_scout_web/lib/block_scout_web/chain.ex +++ b/apps/block_scout_web/lib/block_scout_web/chain.ex @@ -508,6 +508,21 @@ defmodule BlockScoutWeb.Chain do [paging_options: %{@default_paging_options | key: %{block_index: index}}] end + # Clause for `Explorer.Chain.Blackfort.Validator`, + # returned by `BlockScoutWeb.API.V2.ValidatorController.blackfort_validators_list/2` (`/api/v2/validators/blackfort`) + def paging_options(%{ + "address_hash" => address_hash_string + }) do + [ + paging_options: %{ + @default_paging_options + | key: %{ + address_hash: parse_address_hash(address_hash_string) + } + } + ] + end + def paging_options(_params), do: [paging_options: @default_paging_options] def put_key_value_to_paging_options([paging_options: paging_options], key, value) do diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/validator_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/validator_controller.ex index 78736763db..78a59a0ddf 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/validator_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/validator_controller.ex @@ -1,13 +1,15 @@ defmodule BlockScoutWeb.API.V2.ValidatorController do use BlockScoutWeb, :controller - alias Explorer.Chain.Cache.StabilityValidatorsCounters + alias Explorer.Chain.Blackfort.Validator, as: ValidatorBlackfort + alias Explorer.Chain.Cache.{BlackfortValidatorsCounters, StabilityValidatorsCounters} alias Explorer.Chain.Stability.Validator, as: ValidatorStability import BlockScoutWeb.PagingHelper, only: [ delete_parameters_from_next_page_params: 1, stability_validators_state_options: 1, + validators_blackfort_sorting: 1, validators_stability_sorting: 1 ] @@ -71,6 +73,52 @@ defmodule BlockScoutWeb.API.V2.ValidatorController do }) end + @doc """ + Function to handle GET requests to `/api/v2/validators/blackfort` endpoint. + """ + @spec blackfort_validators_list(Plug.Conn.t(), map()) :: Plug.Conn.t() + def blackfort_validators_list(conn, params) do + options = + [ + necessity_by_association: %{ + [address: [:names, :smart_contract, :proxy_implementations]] => :optional + } + ] + |> Keyword.merge(@api_true) + |> Keyword.merge(paging_options(params)) + |> Keyword.merge(validators_blackfort_sorting(params)) + + {validators, next_page} = options |> ValidatorBlackfort.get_paginated_validators() |> split_list_by_page() + + next_page_params = + next_page + |> next_page_params( + validators, + delete_parameters_from_next_page_params(params), + &ValidatorBlackfort.next_page_params/1 + ) + + conn + |> render(:blackfort_validators, %{validators: validators, next_page_params: next_page_params}) + end + + @doc """ + Function to handle GET requests to `/api/v2/validators/blackfort/counters` endpoint. + """ + @spec blackfort_validators_counters(Plug.Conn.t(), map()) :: Plug.Conn.t() + def blackfort_validators_counters(conn, _params) do + %{ + validators_counter: validators_counter, + new_validators_counter: new_validators_counter + } = BlackfortValidatorsCounters.get_counters(@api_true) + + conn + |> json(%{ + validators_counter: validators_counter, + new_validators_counter_24h: new_validators_counter + }) + end + defp calculate_active_validators_percentage(active_validators_counter, validators_counter) do if Decimal.compare(validators_counter, Decimal.new(0)) == :gt do active_validators_counter diff --git a/apps/block_scout_web/lib/block_scout_web/paging_helper.ex b/apps/block_scout_web/lib/block_scout_web/paging_helper.ex index 25dec818db..15bdf142ef 100644 --- a/apps/block_scout_web/lib/block_scout_web/paging_helper.ex +++ b/apps/block_scout_web/lib/block_scout_web/paging_helper.ex @@ -334,4 +334,18 @@ defmodule BlockScoutWeb.PagingHelper do defp do_mud_records_sorting("key1", "asc"), do: [asc_nulls_first: :key1] defp do_mud_records_sorting("key1", "desc"), do: [desc_nulls_last: :key1] defp do_mud_records_sorting(_, _), do: [] + + @spec validators_blackfort_sorting(%{required(String.t()) => String.t()}) :: [ + {:sorting, SortingHelper.sorting_params()} + ] + def validators_blackfort_sorting(%{"sort" => sort_field, "order" => order}) do + [sorting: do_validators_blackfort_sorting(sort_field, order)] + end + + def validators_blackfort_sorting(_), do: [] + + defp do_validators_blackfort_sorting("address_hash", "asc"), do: [asc_nulls_first: :address_hash] + defp do_validators_blackfort_sorting("address_hash", "desc"), do: [desc_nulls_last: :address_hash] + + defp do_validators_blackfort_sorting(_, _), do: [] end diff --git a/apps/block_scout_web/lib/block_scout_web/routers/api_router.ex b/apps/block_scout_web/lib/block_scout_web/routers/api_router.ex index d0e05070af..2a987813fc 100644 --- a/apps/block_scout_web/lib/block_scout_web/routers/api_router.ex +++ b/apps/block_scout_web/lib/block_scout_web/routers/api_router.ex @@ -327,11 +327,21 @@ defmodule BlockScoutWeb.Routers.ApiRouter do end scope "/validators" do - if Application.compile_env(:explorer, :chain_type) == :stability do - scope "/stability" do - get("/", V2.ValidatorController, :stability_validators_list) - get("/counters", V2.ValidatorController, :stability_validators_counters) - end + case Application.compile_env(:explorer, :chain_type) do + :stability -> + scope "/stability" do + get("/", V2.ValidatorController, :stability_validators_list) + get("/counters", V2.ValidatorController, :stability_validators_counters) + end + + :blackfort -> + scope "/blackfort" do + get("/", V2.ValidatorController, :blackfort_validators_list) + get("/counters", V2.ValidatorController, :blackfort_validators_counters) + end + + _ -> + nil end end diff --git a/apps/block_scout_web/lib/block_scout_web/views/api/v2/validator_view.ex b/apps/block_scout_web/lib/block_scout_web/views/api/v2/validator_view.ex index e5b719557b..e54e4f726a 100644 --- a/apps/block_scout_web/lib/block_scout_web/views/api/v2/validator_view.ex +++ b/apps/block_scout_web/lib/block_scout_web/views/api/v2/validator_view.ex @@ -4,14 +4,31 @@ defmodule BlockScoutWeb.API.V2.ValidatorView do alias BlockScoutWeb.API.V2.Helper def render("stability_validators.json", %{validators: validators, next_page_params: next_page_params}) do - %{"items" => Enum.map(validators, &prepare_validator(&1)), "next_page_params" => next_page_params} + %{"items" => Enum.map(validators, &prepare_stability_validator(&1)), "next_page_params" => next_page_params} end - defp prepare_validator(validator) do + def render("blackfort_validators.json", %{validators: validators, next_page_params: next_page_params}) do + %{"items" => Enum.map(validators, &prepare_blackfort_validator(&1)), "next_page_params" => next_page_params} + end + + defp prepare_stability_validator(validator) do %{ "address" => Helper.address_with_info(nil, validator.address, validator.address_hash, true), "state" => validator.state, "blocks_validated_count" => validator.blocks_validated } end + + defp prepare_blackfort_validator(validator) do + %{ + "address" => Helper.address_with_info(nil, validator.address, validator.address_hash, true), + "name" => validator.name, + "commission" => validator.commission, + "self_bonded_amount" => validator.self_bonded_amount, + "delegated_amount" => validator.delegated_amount, + "slashing_status_is_slashed" => validator.slashing_status_is_slashed, + "slashing_status_by_block" => validator.slashing_status_by_block, + "slashing_status_multiplier" => validator.slashing_status_multiplier + } + end end diff --git a/apps/explorer/config/config.exs b/apps/explorer/config/config.exs index 0827fabb05..233186aa47 100644 --- a/apps/explorer/config/config.exs +++ b/apps/explorer/config/config.exs @@ -81,6 +81,11 @@ config :explorer, Explorer.Chain.Cache.StabilityValidatorsCounters, enable_consolidation: true, update_interval_in_milliseconds: update_interval_in_milliseconds_default +config :explorer, Explorer.Chain.Cache.BlackfortValidatorsCounters, + enabled: true, + enable_consolidation: true, + update_interval_in_milliseconds: update_interval_in_milliseconds_default + config :explorer, Explorer.Chain.Cache.TransactionActionTokensData, enabled: true config :explorer, Explorer.Chain.Cache.TransactionActionUniswapPools, enabled: true diff --git a/apps/explorer/config/dev.exs b/apps/explorer/config/dev.exs index 71bfcbe329..08c665e9c4 100644 --- a/apps/explorer/config/dev.exs +++ b/apps/explorer/config/dev.exs @@ -46,6 +46,8 @@ config :explorer, Explorer.Repo.Mud, timeout: :timer.seconds(80) config :explorer, Explorer.Repo.ShrunkInternalTransactions, timeout: :timer.seconds(80) +config :explorer, Explorer.Repo.Blackfort, timeout: :timer.seconds(80) + config :explorer, Explorer.Tracer, env: "dev", disabled?: true config :logger, :explorer, diff --git a/apps/explorer/config/prod.exs b/apps/explorer/config/prod.exs index e273743335..585b7f38df 100644 --- a/apps/explorer/config/prod.exs +++ b/apps/explorer/config/prod.exs @@ -94,6 +94,11 @@ config :explorer, Explorer.Repo.ShrunkInternalTransactions, timeout: :timer.seconds(60), ssl_opts: [verify: :verify_none] +config :explorer, Explorer.Repo.Blackfort, + prepare: :unnamed, + timeout: :timer.seconds(60), + ssl_opts: [verify: :verify_none] + config :explorer, Explorer.Tracer, env: "production", disabled?: true config :logger, :explorer, diff --git a/apps/explorer/config/test.exs b/apps/explorer/config/test.exs index f47582559d..0352d8bc4f 100644 --- a/apps/explorer/config/test.exs +++ b/apps/explorer/config/test.exs @@ -68,7 +68,8 @@ for repo <- [ Explorer.Repo.Filecoin, Explorer.Repo.Stability, Explorer.Repo.Mud, - Explorer.Repo.ShrunkInternalTransactions + Explorer.Repo.ShrunkInternalTransactions, + Explorer.Repo.Blackfort ] do config :explorer, repo, database: database, diff --git a/apps/explorer/lib/explorer/application.ex b/apps/explorer/lib/explorer/application.ex index 4839ed13a8..96685ea646 100644 --- a/apps/explorer/lib/explorer/application.ex +++ b/apps/explorer/lib/explorer/application.ex @@ -144,6 +144,7 @@ defmodule Explorer.Application do configure(Explorer.Migrator.RestoreOmittedWETHTransfers), configure(Explorer.Migrator.FilecoinPendingAddressOperations), configure_mode_dependent_process(Explorer.Migrator.ShrinkInternalTransactions, :indexer), + configure_chain_type_dependent_process(Explorer.Chain.Cache.BlackfortValidatorsCounters, :blackfort), configure_chain_type_dependent_process(Explorer.Chain.Cache.StabilityValidatorsCounters, :stability), configure_mode_dependent_process(Explorer.Migrator.SanitizeMissingTokenBalances, :indexer) ] @@ -168,7 +169,8 @@ defmodule Explorer.Application do Explorer.Repo.BridgedTokens, Explorer.Repo.Filecoin, Explorer.Repo.Stability, - Explorer.Repo.ShrunkInternalTransactions + Explorer.Repo.ShrunkInternalTransactions, + Explorer.Repo.Blackfort ] else [] diff --git a/apps/explorer/lib/explorer/chain/blackfort/validator.ex b/apps/explorer/lib/explorer/chain/blackfort/validator.ex new file mode 100644 index 0000000000..e36a87df63 --- /dev/null +++ b/apps/explorer/lib/explorer/chain/blackfort/validator.ex @@ -0,0 +1,201 @@ +defmodule Explorer.Chain.Blackfort.Validator do + @moduledoc """ + Blackfort validators + """ + + use Explorer.Schema + + alias Explorer.Chain.{Address, Import} + alias Explorer.Chain.Hash.Address, as: HashAddress + alias Explorer.{Chain, Repo, SortingHelper} + + require Logger + + @default_sorting [ + asc: :address_hash + ] + + @primary_key false + typed_schema "validators_blackfort" do + field(:address_hash, HashAddress, primary_key: true) + field(:name, :binary) + field(:commission, :integer) + field(:self_bonded_amount, :decimal) + field(:delegated_amount, :decimal) + field(:slashing_status_is_slashed, :boolean, default: false) + field(:slashing_status_by_block, :integer) + field(:slashing_status_multiplier, :integer) + + has_one(:address, Address, foreign_key: :hash, references: :address_hash) + timestamps() + end + + @required_attrs ~w(address_hash)a + @optional_attrs ~w(name commission self_bonded_amount delegated_amount)a + def changeset(%__MODULE__{} = validator, attrs) do + validator + |> cast(attrs, @required_attrs ++ @optional_attrs) + |> validate_required(@required_attrs) + |> unique_constraint(:address_hash) + end + + @doc """ + Get validators list. + Keyword could contain: + - paging_options + - necessity_by_association + - sorting (supported by `Explorer.SortingHelper` module) + - state (one of `@state_enum`) + """ + @spec get_paginated_validators(keyword()) :: [t()] + def get_paginated_validators(options \\ []) do + paging_options = Keyword.get(options, :paging_options, Chain.default_paging_options()) + necessity_by_association = Keyword.get(options, :necessity_by_association, %{}) + sorting = Keyword.get(options, :sorting, []) + + __MODULE__ + |> Chain.join_associations(necessity_by_association) + |> SortingHelper.apply_sorting(sorting, @default_sorting) + |> SortingHelper.page_with_sorting(paging_options, sorting, @default_sorting) + |> Chain.select_repo(options).all() + end + + @doc """ + Get all validators + """ + @spec get_all_validators(keyword()) :: [t()] + def get_all_validators(options \\ []) do + __MODULE__ + |> Chain.select_repo(options).all() + end + + @doc """ + Delete validators by address hashes + """ + @spec delete_validators_by_address_hashes([binary() | HashAddress.t()]) :: {non_neg_integer(), nil | []} | :ignore + def delete_validators_by_address_hashes(list) when is_list(list) and length(list) > 0 do + __MODULE__ + |> where([vs], vs.address_hash in ^list) + |> Repo.delete_all() + end + + def delete_validators_by_address_hashes(_), do: :ignore + + @doc """ + Insert validators + """ + @spec insert_validators([map()]) :: {non_neg_integer(), nil | []} + def insert_validators(validators) do + Repo.insert_all(__MODULE__, validators, + on_conflict: {:replace_all_except, [:inserted_at]}, + conflict_target: [:address_hash] + ) + end + + @doc """ + Append timestamps (:inserted_at, :updated_at) + """ + @spec append_timestamps(map()) :: map() + def append_timestamps(validator) do + Map.merge(validator, Import.timestamps()) + end + + @doc """ + Derive next page params from %Explorer.Chain.Blackfort.Validator{} + """ + @spec next_page_params(t()) :: map() + def next_page_params(%__MODULE__{address_hash: address_hash}) do + %{"address_hash" => address_hash} + end + + @doc """ + Returns dynamic query for validated blocks count. Needed for SortingHelper + """ + @spec dynamic_validated_blocks() :: Ecto.Query.dynamic_expr() + def dynamic_validated_blocks do + dynamic( + [vs], + fragment( + "SELECT count(*) FROM blocks WHERE miner_hash = ?", + vs.address_hash + ) + ) + end + + @doc """ + Returns total count of validators. + """ + @spec count_validators() :: integer() + def count_validators do + Repo.aggregate(__MODULE__, :count, :address_hash) + end + + @doc """ + Returns count of new validators (inserted withing last 24h). + """ + @spec count_new_validators() :: integer() + def count_new_validators do + __MODULE__ + |> where([vs], vs.inserted_at >= ago(1, "day")) + |> Repo.aggregate(:count, :address_hash) + end + + @doc """ + Fetch list of Blackfort validators + """ + @spec fetch_validators_list() :: {:ok, list()} | :error + def fetch_validators_list do + case HTTPoison.get(validator_url(), [], follow_redirect: true) do + {:ok, %HTTPoison.Response{status_code: 200, body: body}} -> + body |> Jason.decode() |> parse_validators_info() + + error -> + Logger.error("Failed to fetch blackfort validator info: #{inspect(error)}") + :error + end + end + + defp parse_validators_info({:ok, validators}) do + {:ok, + validators + |> Enum.map(fn %{ + "address" => address_hash_string, + "name" => name, + "commission" => commission, + "self_bonded_amount" => self_bonded_amount, + "delegated_amount" => delegated_amount, + "slashing_status" => %{ + "is_slashed" => slashing_status_is_slashed, + "by_block" => slashing_status_by_block, + "multiplier" => slashing_status_multiplier + } + } -> + {:ok, address_hash} = HashAddress.cast(address_hash_string) + + %{ + address_hash: address_hash, + name: name, + commission: parse_number(commission), + self_bonded_amount: parse_number(self_bonded_amount), + delegated_amount: parse_number(delegated_amount), + slashing_status_is_slashed: slashing_status_is_slashed, + slashing_status_by_block: slashing_status_by_block, + slashing_status_multiplier: slashing_status_multiplier + } + end)} + end + + defp parse_validators_info({:error, error}) do + Logger.error("Failed to parse blackfort validator info: #{inspect(error)}") + :error + end + + defp validator_url do + Application.get_env(:explorer, __MODULE__)[:api_url] + end + + defp parse_number(string) do + {number, _} = Integer.parse(string) + number + end +end diff --git a/apps/explorer/lib/explorer/chain/cache/blackfort_validators_counters.ex b/apps/explorer/lib/explorer/chain/cache/blackfort_validators_counters.ex new file mode 100644 index 0000000000..cf2a774111 --- /dev/null +++ b/apps/explorer/lib/explorer/chain/cache/blackfort_validators_counters.ex @@ -0,0 +1,97 @@ +defmodule Explorer.Chain.Cache.BlackfortValidatorsCounters do + @moduledoc """ + Counts and store counters of validators blackfort. + + It loads the count asynchronously and in a time interval of 30 minutes. + """ + + use GenServer + + alias Explorer.Chain + alias Explorer.Chain.Blackfort.Validator, as: ValidatorBlackfort + + @validators_counter_key "blackfort_validators_counter" + @new_validators_counter_key "new_blackfort_validators_counter" + + # It is undesirable to automatically start the consolidation in all environments. + # Consider the test environment: if the consolidation initiates but does not + # finish before a test ends, that test will fail. This way, hundreds of + # tests were failing before disabling the consolidation and the scheduler in + # the test env. + config = Application.compile_env(:explorer, __MODULE__) + @enable_consolidation Keyword.get(config, :enable_consolidation) + + @update_interval_in_milliseconds Keyword.get(config, :update_interval_in_milliseconds) + + @doc """ + Starts a process to periodically update validators blackfort counters + """ + @spec start_link(term()) :: GenServer.on_start() + def start_link(_) do + GenServer.start_link(__MODULE__, [], name: __MODULE__) + end + + @impl true + def init(_args) do + {:ok, %{consolidate?: @enable_consolidation}, {:continue, :ok}} + end + + defp schedule_next_consolidation do + Process.send_after(self(), :consolidate, @update_interval_in_milliseconds) + end + + @impl true + def handle_continue(:ok, %{consolidate?: true} = state) do + consolidate() + schedule_next_consolidation() + + {:noreply, state} + end + + @impl true + def handle_continue(:ok, state) do + {:noreply, state} + end + + @impl true + def handle_info(:consolidate, state) do + consolidate() + schedule_next_consolidation() + + {:noreply, state} + end + + @doc """ + Fetches values for a blackfort validators counters from the `last_fetched_counters` table. + """ + @spec get_counters(Keyword.t()) :: map() + def get_counters(options) do + %{ + validators_counter: Chain.get_last_fetched_counter(@validators_counter_key, options), + new_validators_counter: Chain.get_last_fetched_counter(@new_validators_counter_key, options) + } + end + + @doc """ + Consolidates the info by populating the `last_fetched_counters` table with the current database information. + """ + @spec consolidate() :: any() + def consolidate do + tasks = [ + Task.async(fn -> ValidatorBlackfort.count_validators() end), + Task.async(fn -> ValidatorBlackfort.count_new_validators() end) + ] + + [validators_counter, new_validators_counter] = Task.await_many(tasks, :infinity) + + Chain.upsert_last_fetched_counter(%{ + counter_type: @validators_counter_key, + value: validators_counter + }) + + Chain.upsert_last_fetched_counter(%{ + counter_type: @new_validators_counter_key, + value: new_validators_counter + }) + end +end diff --git a/apps/explorer/lib/explorer/repo.ex b/apps/explorer/lib/explorer/repo.ex index 08a2a924be..548165168e 100644 --- a/apps/explorer/lib/explorer/repo.ex +++ b/apps/explorer/lib/explorer/repo.ex @@ -286,4 +286,14 @@ defmodule Explorer.Repo do ConfigHelper.init_repo_module(__MODULE__, opts) end end + + defmodule Blackfort do + use Ecto.Repo, + otp_app: :explorer, + adapter: Ecto.Adapters.Postgres + + def init(_, opts) do + ConfigHelper.init_repo_module(__MODULE__, opts) + end + end end diff --git a/apps/explorer/priv/blackfort/migrations/20240910112251_add_blackfort_validators.exs b/apps/explorer/priv/blackfort/migrations/20240910112251_add_blackfort_validators.exs new file mode 100644 index 0000000000..d1d2b55e8d --- /dev/null +++ b/apps/explorer/priv/blackfort/migrations/20240910112251_add_blackfort_validators.exs @@ -0,0 +1,20 @@ +defmodule Explorer.Repo.Blackfort.Migrations.AddBlackfortValidators do + use Ecto.Migration + + def change do + create table(:validators_blackfort, primary_key: false) do + add(:address_hash, :bytea, null: false, primary_key: true) + add(:name, :string) + add(:commission, :smallint) + add(:self_bonded_amount, :numeric, precision: 100) + add(:delegated_amount, :numeric, precision: 100) + add(:slashing_status_is_slashed, :boolean, default: false) + add(:slashing_status_by_block, :bigint) + add(:slashing_status_multiplier, :integer) + + timestamps() + end + + create_if_not_exists(index(:validators_blackfort, ["address_hash ASC"])) + end +end diff --git a/apps/indexer/lib/indexer/block/realtime/fetcher.ex b/apps/indexer/lib/indexer/block/realtime/fetcher.ex index a2e7716ac1..598a248655 100644 --- a/apps/indexer/lib/indexer/block/realtime/fetcher.ex +++ b/apps/indexer/lib/indexer/block/realtime/fetcher.ex @@ -170,14 +170,21 @@ defmodule Indexer.Block.Realtime.Fetcher do Process.cancel_timer(timer) end - if Application.compile_env(:explorer, :chain_type) == :stability do - defp fetch_validators_async do - GenServer.cast(Indexer.Fetcher.Stability.Validator, :update_validators_list) - end - else - defp fetch_validators_async do - :ignore - end + case Application.compile_env(:explorer, :chain_type) do + :stability -> + defp fetch_validators_async do + GenServer.cast(Indexer.Fetcher.Stability.Validator, :update_validators_list) + end + + :blackfort -> + defp fetch_validators_async do + GenServer.cast(Indexer.Fetcher.Blackfort.Validator, :update_validators_list) + end + + _ -> + defp fetch_validators_async do + :ignore + end end defp subscribe_to_new_heads(%__MODULE__{subscription: nil} = state, subscribe_named_arguments) diff --git a/apps/indexer/lib/indexer/fetcher/blackfort/validator.ex b/apps/indexer/lib/indexer/fetcher/blackfort/validator.ex new file mode 100644 index 0000000000..75cbc791d1 --- /dev/null +++ b/apps/indexer/lib/indexer/fetcher/blackfort/validator.ex @@ -0,0 +1,49 @@ +defmodule Indexer.Fetcher.Blackfort.Validator do + @moduledoc """ + GenServer responsible for updating the list of blackfort validators in the database. + """ + use GenServer + + alias Explorer.Chain.Blackfort.Validator + + def start_link(_) do + GenServer.start_link(__MODULE__, [], name: __MODULE__) + end + + @impl true + def init(state) do + GenServer.cast(__MODULE__, :update_validators_list) + + {:ok, state} + end + + @impl true + def handle_cast(:update_validators_list, state) do + case Validator.fetch_validators_list() do + {:ok, validators} -> + validators_from_db = Validator.get_all_validators() + + validators_map = + Enum.reduce(validators, %{}, fn %{address_hash: address_hash}, map -> + Map.put(map, address_hash.bytes, true) + end) + + address_hashes_to_drop_from_db = + Enum.flat_map(validators_from_db, fn validator -> + (is_nil(validators_map[validator.address_hash.bytes]) && + [validator.address_hash]) || [] + end) + + Validator.delete_validators_by_address_hashes(address_hashes_to_drop_from_db) + + validators + |> Enum.map(&Validator.append_timestamps/1) + |> Validator.insert_validators() + + _ -> + nil + end + + {:noreply, state} + end +end diff --git a/apps/indexer/lib/indexer/supervisor.ex b/apps/indexer/lib/indexer/supervisor.ex index e56883914c..70ea093532 100644 --- a/apps/indexer/lib/indexer/supervisor.ex +++ b/apps/indexer/lib/indexer/supervisor.ex @@ -18,6 +18,7 @@ defmodule Indexer.Supervisor do alias Indexer.Block.Catchup, as: BlockCatchup alias Indexer.Block.Realtime, as: BlockRealtime + alias Indexer.Fetcher.Blackfort.Validator, as: ValidatorBlackfort alias Indexer.Fetcher.CoinBalance.Catchup, as: CoinBalanceCatchup alias Indexer.Fetcher.CoinBalance.Realtime, as: CoinBalanceRealtime alias Indexer.Fetcher.Stability.Validator, as: ValidatorStability @@ -285,6 +286,9 @@ defmodule Indexer.Supervisor do :stability -> [{ValidatorStability, []} | fetchers] + :blackfort -> + [{ValidatorBlackfort, []} | fetchers] + _ -> fetchers end diff --git a/config/config_helper.exs b/config/config_helper.exs index 540f887a0a..6def45a00a 100644 --- a/config/config_helper.exs +++ b/config/config_helper.exs @@ -23,6 +23,7 @@ defmodule ConfigHelper do :zksync -> base_repos ++ [Explorer.Repo.ZkSync] :celo -> base_repos ++ [Explorer.Repo.Celo] :arbitrum -> base_repos ++ [Explorer.Repo.Arbitrum] + :blackfort -> base_repos ++ [Explorer.Repo.Blackfort] _ -> base_repos end @@ -316,7 +317,8 @@ defmodule ConfigHelper do "suave", "zetachain", "zksync", - "celo" + "celo", + "blackfort" ] @spec chain_type() :: atom() | nil diff --git a/config/runtime.exs b/config/runtime.exs index 756bb1ef2a..7875f4c476 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -646,6 +646,8 @@ config :explorer, Explorer.Migrator.FilecoinPendingAddressOperations, batch_size: ConfigHelper.parse_integer_env_var("FILECOIN_PENDING_ADDRESS_OPERATIONS_MIGRATION_BATCH_SIZE", 100), concurrency: ConfigHelper.parse_integer_env_var("FILECOIN_PENDING_ADDRESS_OPERATIONS_MIGRATION_CONCURRENCY", 1) +config :explorer, Explorer.Chain.Blackfort.Validator, api_url: System.get_env("BLACKFORT_VALIDATOR_API_URL") + ############### ### Indexer ### ############### diff --git a/config/runtime/dev.exs b/config/runtime/dev.exs index cdd35cc636..0e2d20cc0d 100644 --- a/config/runtime/dev.exs +++ b/config/runtime/dev.exs @@ -199,6 +199,13 @@ config :explorer, Explorer.Repo.ShrunkInternalTransactions, url: System.get_env("DATABASE_URL"), pool_size: 1 +# Configures Blackfort database +config :explorer, Explorer.Repo.Blackfort, + database: database, + hostname: hostname, + url: System.get_env("DATABASE_URL"), + pool_size: 1 + variant = Variant.get() Code.require_file("#{variant}.exs", "apps/explorer/config/dev") diff --git a/config/runtime/prod.exs b/config/runtime/prod.exs index 2b10f79f28..f52ef52b59 100644 --- a/config/runtime/prod.exs +++ b/config/runtime/prod.exs @@ -159,6 +159,12 @@ config :explorer, Explorer.Repo.ShrunkInternalTransactions, pool_size: 1, ssl: ExplorerConfigHelper.ssl_enabled?() +# Configures Blackfort database +config :explorer, Explorer.Repo.Blackfort, + url: System.get_env("DATABASE_URL"), + pool_size: 1, + ssl: ExplorerConfigHelper.ssl_enabled?() + variant = Variant.get() Code.require_file("#{variant}.exs", "apps/explorer/config/prod") diff --git a/cspell.json b/cspell.json index be5ffa2cca..22872a9c56 100644 --- a/cspell.json +++ b/cspell.json @@ -18,8 +18,8 @@ "AION", "AIRTABLE", "Aiubo", - "alloc", "alfajores", + "alloc", "amzootyukbugmx", "anytrust", "apikey", @@ -54,6 +54,7 @@ "binwrite", "bitmask", "bizbuz", + "Blackfort", "Blockchair", "blockheight", "blockless", @@ -200,9 +201,9 @@ "falala", "feelin", "FEVM", + "filecoin", "Filecoin", "Filesize", - "filecoin", "fixidity", "fkey", "Floki",