From 6c072464466f3ed51dbb437ef62fb0ccea594c11 Mon Sep 17 00:00:00 2001 From: Fedor Ivanov Date: Tue, 6 Aug 2024 19:40:49 +0300 Subject: [PATCH] feat: support for `:celo` chain type (#10564) * feat: indexing of Celo transactions (#10275) * feat(json-rpc): support celo-specific fields in transactions * chore: add new chain type to matrix builder * feat: add celo-specific fields to transactions schema * chore: add "celo" to `cspell.json` * chore: fix formatting * refactor: improve naming for celo fields and ignore `ethCompatible` field * chore: add block fetcher test for celo fields * feat: add celo core contract abis * feat: add a cache for core celo contracts * feat: treat celo native token transfers as token transfers * fix: update `cspell.json` with celo words * fix(dialyzer): pattern can never match the type * fix: update `cspell.json` with celo words * feat: move `CELO_NETWORK` env to config * feat: parse native Celo token transfers from internal transactions * feat: add token balances fetching * refactor: remove `dbg` and organize code better * fix: start TokenBalance fetcher * fix: start token balance fetcher * fix: typo in log message * fix: missing fields in celo transaction token transfer * refactor: remove `AbiHandler` in favour of hardcoded pieces of abi * refactor: native token transfers from internal transactions fether * feat: implement parsing of token transfers for traceable rpc node * feat: fetch address coin balances after erc20 celo token transfers * feat: add `gas_token` to transaction response in api v2 * feat: index tokens specified as `feeCurrency` * chore: fix credo warnings * chore: decrease log level to debug * chore: fix cspell warnings * refactor: apply suggestions * fix: handle case when `gas_token` is not loaded * dbg * chore: expect contract calls in celo fetcher test * fix: compile errors for celo view on default chain type * chore: remove unused import * refactor: move `celo_gas_tokens` to chain type imports * refactor: apply suggestions * fix: swap arguments in `put_if_present` * fix: indexer tests * refactor: always include `gas_token` to the response * feat: add `gas_token` to all transaction-related endpoints * feat: celo core contracts with historical data * fix: `uncles` field is expected but not present in RPC node response * fix: credo & format * fix: define `CELO_CORE_CONTRACTS` in tests * fix: put `gas_token` under `celo` section * refactor: move token transfers filtering from `TokenTotalSupplyUpdater` to `Transform.TokenTransfers` * feat: add workflow to publish docker image * refactor: apply suggestions by @nikitosing * chore: add docs and specs * fix: malformed import * fix: publish image for celo * fix: ci build for celo * feat: index celo epoch transactions (#9944) * feat(json-rpc): support celo-specific fields in transactions * chore: add new chain type to matrix builder * feat: add celo-specific fields to transactions schema * chore: add "celo" to `cspell.json` * chore: fix formatting * refactor: improve naming for celo fields and ignore `ethCompatible` field * chore: add block fetcher test for celo fields * feat: add celo core contract abis * feat: add a cache for core celo contracts * feat: treat celo native token transfers as token transfers * fix: update `cspell.json` with celo words * fix(dialyzer): pattern can never match the type * fix: update `cspell.json` with celo words * feat: move `CELO_NETWORK` env to config * feat: parse native Celo token transfers from internal transactions * feat: add token balances fetching * refactor: remove `dbg` and organize code better * fix: start TokenBalance fetcher * fix: start token balance fetcher * fix: typo in log message * fix: missing fields in celo transaction token transfer * refactor: remove `AbiHandler` in favour of hardcoded pieces of abi * refactor: native token transfers from internal transactions fether * feat: implement parsing of token transfers for traceable rpc node * feat: fetch address coin balances after erc20 celo token transfers * feat: add `gas_token` to transaction response in api v2 * feat: index tokens specified as `feeCurrency` * chore: fix credo warnings * chore: decrease log level to debug * chore: fix cspell warnings * refactor: apply suggestions * fix: handle case when `gas_token` is not loaded * dbg * chore: expect contract calls in celo fetcher test * fix: compile errors for celo view on default chain type * chore: remove unused import * refactor: move `celo_gas_tokens` to chain type imports * refactor: apply suggestions * fix: swap arguments in `put_if_present` * fix: indexer tests * refactor: always include `gas_token` to the response * feat: add `gas_token` to all transaction-related endpoints * feat: celo core contracts with historical data * fix: `uncles` field is expected but not present in RPC node response * fix: credo & format * fix: define `CELO_CORE_CONTRACTS` in tests * fix: put `gas_token` under `celo` section * refactor: move token transfers filtering from `TokenTotalSupplyUpdater` to `Transform.TokenTransfers` * feat: add workflow to publish docker image * refactor: apply suggestions by @nikitosing * chore: add docs and specs * fix: malformed import * feat: add pending epoch operations table * refactor: fix format * feat: add transformers for epoch events * feat: add query to stream pending epoch block operations * fix: call to renamed function * fix: add factory method for pending epoch block operation * feat: add utils for logs parsing * feat: add schemas for epoch rewards and election rewards * fix: rename transformer according to event name * feat: implement epoch rewards fetcher * feat: fetch and import epoch logs * feat: improve epoch rewards fetcher 1. Do not fetch logs -- use the ones stored in DB 2. Import epochs to the database 3. Configure fetcher in runtime.exs * feat: add epoch helper functions * activated_validator_group_vote.ex * feat: add task to fetch core contracts * feat: fetch core contract events in the task * fix: merge artifacts * refactor: logs requests and reduce scope of epoch logs request * chore: fix formatting, credo warning, etc. * feat: fetch epoch rewards in one sql query * feat: fetch validator group votes (historical data and on demand) * refactor: rename fields and replace all entries on conflict * feat: validator group votes fetcher * fix: put each topic in a separate request * feat: fetch voter rewards * refactor: split epoch fetcher to multiple modules * feat: send epoch blocks for async fetching from block fetcher * chore: fix credo and formatting issues * fix: dialyzer warnings * fix: add on demand fetch of group votes * fix: failed explorer and indexer tests * fix: block fetcher tests * fix: match error in epoch logs * feat: add env to manage logs batch size when fetching validator group votes * fix: Add `ssl_opts` for Celo repo * fix: add `disabled?` predicate to supervisor config * fix: return empty list when `getPaymentDelegation` is not available * fix: validate the case when there is no voter rewards for an epoch * fix: formatting * fix: import `put_if_present/3` * fix: do not treat genesis block as an epoch block * fix: some of the fetcher tests * dbg * chore: remove commented code * chore: canonical disable flag env name * fix: run test only for celo chain type * fix: add explicit `wait_ for_tasks` in token instance realtime test * fix: set missing `CELO_CORE_CONTRACTS` env * fix: rollback token instance realtime test * chore: remove debug artifact * chore: add docs for new tables * perf: remove unused fields from `celo_validator_group_votes` table * fix: add on_exit clause * refactor: use `remove/3` function for migration rollback possibility * refactor: extract celo core contracts environment variable setup into a separate function * chore: add new vars to `common-blockscout.env` file * chore: add specs and docs for new modules and functions * refactor: eliminate unused import warnings * fix: credo warning * feat: API for celo epoch rewards and fees breakdown (#10308) * feat(json-rpc): support celo-specific fields in transactions * chore: add new chain type to matrix builder * feat: add celo-specific fields to transactions schema * chore: add "celo" to `cspell.json` * chore: fix formatting * refactor: improve naming for celo fields and ignore `ethCompatible` field * chore: add block fetcher test for celo fields * feat: add celo core contract abis * feat: add a cache for core celo contracts * feat: treat celo native token transfers as token transfers * fix: update `cspell.json` with celo words * fix(dialyzer): pattern can never match the type * fix: update `cspell.json` with celo words * feat: move `CELO_NETWORK` env to config * feat: parse native Celo token transfers from internal transactions * feat: add token balances fetching * refactor: remove `dbg` and organize code better * fix: start TokenBalance fetcher * fix: start token balance fetcher * fix: typo in log message * fix: missing fields in celo transaction token transfer * refactor: remove `AbiHandler` in favour of hardcoded pieces of abi * refactor: native token transfers from internal transactions fether * feat: implement parsing of token transfers for traceable rpc node * feat: fetch address coin balances after erc20 celo token transfers * feat: add `gas_token` to transaction response in api v2 * feat: index tokens specified as `feeCurrency` * chore: fix credo warnings * chore: decrease log level to debug * chore: fix cspell warnings * refactor: apply suggestions * fix: handle case when `gas_token` is not loaded * dbg * chore: expect contract calls in celo fetcher test * fix: compile errors for celo view on default chain type * chore: remove unused import * refactor: move `celo_gas_tokens` to chain type imports * refactor: apply suggestions * fix: swap arguments in `put_if_present` * fix: indexer tests * feat: add pending epoch operations table * feat: add several celo core contracts * refactor: fix format * feat: add new core contract default addresses * feat: add transformers for epoch events * feat: add query to stream pending epoch block operations * fix: call to renamed function * fix: add factory method for pending epoch block operation * feat: add utils for logs parsing * feat: add schemas for epoch rewards and election rewards * fix: rename transformer according to event name * feat: implement epoch rewards fetcher * feat: fetch and import epoch logs * feat: improve epoch rewards fetcher 1. Do not fetch logs -- use the ones stored in DB 2. Import epochs to the database 3. Configure fetcher in runtime.exs * fix: remove duplicated module attribute * feat: add epoch helper functions * activated_validator_group_vote.ex * feat: tmp * refactor: always include `gas_token` to the response * feat: add `gas_token` to all transaction-related endpoints * feat: celo core contracts with historical data * fix: `uncles` field is expected but not present in RPC node response * fix: credo & format * fix: define `CELO_CORE_CONTRACTS` in tests * fix: put `gas_token` under `celo` section * refactor: move token transfers filtering from `TokenTotalSupplyUpdater` to `Transform.TokenTransfers` * feat: add workflow to publish docker image * refactor: apply suggestions by @nikitosing * chore: add docs and specs * fix: malformed import * feat: add pending epoch operations table * refactor: fix format * feat: add transformers for epoch events * feat: add query to stream pending epoch block operations * fix: call to renamed function * fix: add factory method for pending epoch block operation * feat: add utils for logs parsing * feat: add schemas for epoch rewards and election rewards * fix: rename transformer according to event name * feat: implement epoch rewards fetcher * feat: fetch and import epoch logs * feat: improve epoch rewards fetcher 1. Do not fetch logs -- use the ones stored in DB 2. Import epochs to the database 3. Configure fetcher in runtime.exs * feat: add epoch helper functions * activated_validator_group_vote.ex * feat: add task to fetch core contracts * feat: fetch core contract events in the task * fix: merge artifacts * refactor: logs requests and reduce scope of epoch logs request * chore: fix formatting, credo warning, etc. * feat: fetch epoch rewards in one sql query * feat: fetch validator group votes (historical data and on demand) * refactor: rename fields and replace all entries on conflict * feat: validator group votes fetcher * fix: put each topic in a separate request * feat: fetch voter rewards * fix: merge artifacts * feat: add epoch rewards in block api response * fix: merge artifacts * feat: add base fee breakdown in block api response * feat: add aggregated election rewards to block api response * perf: use replica when querying epoch rewards * feat: endpoint for paginated election rewards * feat: endpoint for paginated election rewards for address * chore: rename `rewards` to `distributions` * fix: add missing reward type to API error message * refactor: split epoch fetcher to multiple modules * feat: send epoch blocks for async fetching from block fetcher * chore: fix credo and formatting issues * fix: dialyzer warnings * fix: add on demand fetch of group votes * fix: failed explorer and indexer tests * fix: block fetcher tests * chore: remove unused module * fix: sort election rewards by block number * fix: match error in epoch logs * feat: add env to manage logs batch size when fetching validator group votes * fix: credo and formatting warnings * fix: cspell errors * fix: dialyzer errors * fix: Add `ssl_opts` for Celo repo * fix: add missing preloads and make more explicit api response for block * fix: add `disabled?` predicate to supervisor config * fix: return empty list when `getPaymentDelegation` is not available * fix: validate the case when there is no voter rewards for an epoch * fix: formatting * fix: add missing preload * refactor: more robust fees breakdown logic for the case of fee handler * fix: formatting * refactor: move epoch information to the separate endpoint * fix: import `put_if_present/3` * fix: formatting * fix: do not treat genesis block as an epoch block * fix: some of the fetcher tests * dbg * chore: remove commented code * chore: canonical disable flag env name * fix: run test only for celo chain type * fix: add explicit `wait_ for_tasks` in token instance realtime test * fix: set missing `CELO_CORE_CONTRACTS` env * fix: rollback token instance realtime test * chore: remove debug artifact * chore: add `@docs` and `@specs` * fix: missing core contracts var in tests * feat: extend `tabs-counters` endpoint with election rewards count * refactor: move dead address to `Explorer.Chain.SmartContract` * perf: clause with simplified queries in the case of `amount == 0` and/or `block_number == 0` * fix: it comes that `NotLoaded` clause is not redundant actually... * fix: add missing preload in `BlockScoutWeb.AddressChannel` * chore: unset `CELO_CORE_CONTRACTS` in tests * chore: add missing specs * fix: `paginate` clause when `amount == 0` * refactor: move `paging_options/2` to `Explorer.Chain.Celo.ElectionReward` * refactor: avoid using virtual field for block number. * fix: remove redundant condition in the query * chore: clarify `@spec` * chore: fix credo warning * chore: remove `fi-celo` from branches that trigger ci * fix: merge artifacts * fix: remove todo comment --- .github/workflows/config.yml | 2 +- .../publish-docker-image-for-celo.yml | 48 ++ .../lib/block_scout_web/chain.ex | 17 +- .../channels/address_channel.ex | 25 +- .../controllers/api/v2/address_controller.ex | 116 +++- .../api/v2/advanced_filter_controller.ex | 3 +- .../controllers/api/v2/block_controller.ex | 136 +++- .../controllers/api/v2/fallback_controller.ex | 24 +- .../api/v2/main_page_controller.ex | 24 +- .../controllers/api/v2/mud_controller.ex | 4 +- .../controllers/api/v2/token_controller.ex | 4 +- .../api/v2/transaction_controller.ex | 5 + .../models/transaction_state_helper.ex | 2 +- .../lib/block_scout_web/routers/api_router.ex | 9 + .../views/api/v2/block_view.ex | 6 + .../block_scout_web/views/api/v2/celo_view.ex | 391 +++++++++++ .../views/api/v2/transaction_view.ex | 10 + .../api/v2/block_controller_test.exs | 22 + .../api/v2/transaction_controller_test.exs | 179 +++++ apps/ethereum_jsonrpc/lib/ethereum_jsonrpc.ex | 12 + .../lib/ethereum_jsonrpc/block.ex | 43 +- .../lib/ethereum_jsonrpc/geth/call.ex | 3 +- .../lib/ethereum_jsonrpc/logs.ex | 106 ++- .../lib/ethereum_jsonrpc/nethermind/trace.ex | 2 +- .../lib/ethereum_jsonrpc/transaction.ex | 53 +- apps/explorer/config/dev.exs | 2 + apps/explorer/config/prod.exs | 5 + apps/explorer/config/test.exs | 1 + .../account/notifier/forbidden_address.ex | 8 +- apps/explorer/lib/explorer/application.ex | 1 + apps/explorer/lib/explorer/chain.ex | 4 +- .../lib/explorer/chain/address/counters.ex | 23 +- apps/explorer/lib/explorer/chain/block.ex | 24 +- .../chain/cache/celo_core_contracts.ex | 235 +++++++ .../explorer/chain/celo/election_reward.ex | 455 +++++++++++++ .../lib/explorer/chain/celo/epoch_reward.ex | 117 ++++ .../lib/explorer/chain/celo/helper.ex | 94 +++ .../celo/pending_epoch_block_operation.ex | 74 ++ .../lib/explorer/chain/celo/reader.ex | 193 ++++++ .../chain/celo/validator_group_vote.ex | 76 +++ .../explorer/chain/import/runner/blocks.ex | 44 ++ .../import/runner/celo/election_rewards.ex | 93 +++ .../chain/import/runner/celo/epoch_rewards.ex | 139 ++++ .../runner/celo/validator_group_votes.ex | 87 +++ .../lib/explorer/chain/import/runner/logs.ex | 99 ++- .../chain/import/runner/token_transfers.ex | 106 ++- .../chain/import/runner/transactions.ex | 78 +++ .../chain/import/stage/block_referencing.ex | 139 ++-- apps/explorer/lib/explorer/chain/log.ex | 128 +++- .../lib/explorer/chain/smart_contract.ex | 11 +- .../explorer/chain/smart_contract/proxy.ex | 2 +- .../lib/explorer/chain/token_transfer.ex | 188 +++-- .../lib/explorer/chain/transaction.ex | 25 + apps/explorer/lib/explorer/helper.ex | 36 +- apps/explorer/lib/explorer/paging_options.ex | 10 + apps/explorer/lib/explorer/repo.ex | 10 + .../explorer/lib/fetch_celo_core_contracts.ex | 236 +++++++ .../20240323152023_add_custom_fields.exs | 11 + ...856_add_pending_epoch_block_operations.exs | 14 + ...nsaction_hash_from_primary_key_in_logs.exs | 26 + ...sh_from_primary_key_in_token_transfers.exs | 23 + .../20240607185817_add_epoch_rewards.exs | 62 ++ ...240612135216_add_validator_group_votes.exs | 36 + .../20240614125614_add_election_rewards.exs | 38 ++ ...used_fields_from_validator_group_votes.exs | 10 + .../chain/cache/celo_core_contracts_test.exs | 84 +++ .../chain/import/runner/blocks_test.exs | 57 ++ apps/explorer/test/support/factory.ex | 5 + .../lib/indexer/block/catchup/fetcher.ex | 4 +- apps/indexer/lib/indexer/block/fetcher.ex | 42 +- .../lib/indexer/block/realtime/fetcher.ex | 8 +- .../fetcher/celo/epoch_block_operations.ex | 146 ++++ .../core_contract_version.ex | 92 +++ .../delegated_payments.ex | 166 +++++ .../epoch_block_operations/distributions.ex | 101 +++ .../validator_and_group_payments.ex | 67 ++ .../epoch_block_operations/voter_payments.ex | 201 ++++++ .../lib/indexer/fetcher/celo/epoch_logs.ex | 121 ++++ .../lib/indexer/fetcher/celo/helper.ex | 29 + .../fetcher/celo/validator_group_votes.ex | 217 ++++++ .../indexer/fetcher/internal_transaction.ex | 59 +- apps/indexer/lib/indexer/helper.ex | 109 ++- apps/indexer/lib/indexer/supervisor.ex | 7 + .../transform/address_coin_balances.ex | 40 +- .../lib/indexer/transform/addresses.ex | 36 +- .../transform/celo/transaction_gas_tokens.ex | 42 ++ .../celo/transaction_token_transfers.ex | 138 ++++ .../validator_epoch_payment_distributions.ex | 72 ++ .../lib/indexer/transform/token_transfers.ex | 19 +- .../bound_interval_supervisor_test.exs | 29 + .../indexer/block/catchup/fetcher_test.exs | 29 +- .../test/indexer/block/fetcher_test.exs | 643 ++++++++++++++---- .../indexer/block/realtime/fetcher_test.exs | 144 +++- .../celo/epoch_block_operations_test.exs | 46 ++ .../fetcher/internal_transaction_test.exs | 19 +- .../celo_epoch_rewards_supervisor_case.ex | 17 + config/config_helper.exs | 6 +- config/runtime.exs | 16 + config/runtime/dev.exs | 9 + config/runtime/prod.exs | 8 + cspell.json | 10 +- docker-compose/envs/common-blockscout.env | 5 +- 102 files changed, 6495 insertions(+), 557 deletions(-) create mode 100644 .github/workflows/publish-docker-image-for-celo.yml create mode 100644 apps/block_scout_web/lib/block_scout_web/views/api/v2/celo_view.ex create mode 100644 apps/explorer/lib/explorer/chain/cache/celo_core_contracts.ex create mode 100644 apps/explorer/lib/explorer/chain/celo/election_reward.ex create mode 100644 apps/explorer/lib/explorer/chain/celo/epoch_reward.ex create mode 100644 apps/explorer/lib/explorer/chain/celo/helper.ex create mode 100644 apps/explorer/lib/explorer/chain/celo/pending_epoch_block_operation.ex create mode 100644 apps/explorer/lib/explorer/chain/celo/reader.ex create mode 100644 apps/explorer/lib/explorer/chain/celo/validator_group_vote.ex create mode 100644 apps/explorer/lib/explorer/chain/import/runner/celo/election_rewards.ex create mode 100644 apps/explorer/lib/explorer/chain/import/runner/celo/epoch_rewards.ex create mode 100644 apps/explorer/lib/explorer/chain/import/runner/celo/validator_group_votes.ex create mode 100644 apps/explorer/lib/fetch_celo_core_contracts.ex create mode 100644 apps/explorer/priv/celo/migrations/20240323152023_add_custom_fields.exs create mode 100644 apps/explorer/priv/celo/migrations/20240424121856_add_pending_epoch_block_operations.exs create mode 100644 apps/explorer/priv/celo/migrations/20240512143204_remove_transaction_hash_from_primary_key_in_logs.exs create mode 100644 apps/explorer/priv/celo/migrations/20240513091316_remove_transaction_hash_from_primary_key_in_token_transfers.exs create mode 100644 apps/explorer/priv/celo/migrations/20240607185817_add_epoch_rewards.exs create mode 100644 apps/explorer/priv/celo/migrations/20240612135216_add_validator_group_votes.exs create mode 100644 apps/explorer/priv/celo/migrations/20240614125614_add_election_rewards.exs create mode 100644 apps/explorer/priv/celo/migrations/20240715110334_remove_unused_fields_from_validator_group_votes.exs create mode 100644 apps/explorer/test/explorer/chain/cache/celo_core_contracts_test.exs create mode 100644 apps/indexer/lib/indexer/fetcher/celo/epoch_block_operations.ex create mode 100644 apps/indexer/lib/indexer/fetcher/celo/epoch_block_operations/core_contract_version.ex create mode 100644 apps/indexer/lib/indexer/fetcher/celo/epoch_block_operations/delegated_payments.ex create mode 100644 apps/indexer/lib/indexer/fetcher/celo/epoch_block_operations/distributions.ex create mode 100644 apps/indexer/lib/indexer/fetcher/celo/epoch_block_operations/validator_and_group_payments.ex create mode 100644 apps/indexer/lib/indexer/fetcher/celo/epoch_block_operations/voter_payments.ex create mode 100644 apps/indexer/lib/indexer/fetcher/celo/epoch_logs.ex create mode 100644 apps/indexer/lib/indexer/fetcher/celo/helper.ex create mode 100644 apps/indexer/lib/indexer/fetcher/celo/validator_group_votes.ex create mode 100644 apps/indexer/lib/indexer/transform/celo/transaction_gas_tokens.ex create mode 100644 apps/indexer/lib/indexer/transform/celo/transaction_token_transfers.ex create mode 100644 apps/indexer/lib/indexer/transform/celo/validator_epoch_payment_distributions.ex create mode 100644 apps/indexer/test/indexer/fetcher/celo/epoch_block_operations_test.exs create mode 100644 apps/indexer/test/support/indexer/fetcher/celo_epoch_rewards_supervisor_case.ex diff --git a/.github/workflows/config.yml b/.github/workflows/config.yml index 7a852ee6d0..edefb0f392 100644 --- a/.github/workflows/config.yml +++ b/.github/workflows/config.yml @@ -49,7 +49,7 @@ jobs: // Add/remove CI matrix chain types here const defaultChainTypes = ["default"]; - const chainTypes = ["ethereum", "polygon_zkevm", "rsk", "stability", "filecoin", "optimism", "arbitrum"]; + const chainTypes = ["ethereum", "polygon_zkevm", "rsk", "stability", "filecoin", "optimism", "arbitrum", "celo"]; const extraChainTypes = ["suave", "polygon_edge"]; // Chain type matrix we use in master branch diff --git a/.github/workflows/publish-docker-image-for-celo.yml b/.github/workflows/publish-docker-image-for-celo.yml new file mode 100644 index 0000000000..ace16f10ed --- /dev/null +++ b/.github/workflows/publish-docker-image-for-celo.yml @@ -0,0 +1,48 @@ +name: Celo Publish Docker image + +on: + workflow_dispatch: + push: + branches: + - production-celo +jobs: + push_to_registry: + name: Push Docker image to Docker Hub + runs-on: ubuntu-latest + env: + RELEASE_VERSION: ${{ vars.RELEASE_VERSION }} + DOCKER_CHAIN_NAME: celo + steps: + - uses: actions/checkout@v4 + - name: Setup repo + uses: ./.github/actions/setup-repo + id: setup + with: + docker-username: ${{ secrets.DOCKER_USERNAME }} + docker-password: ${{ secrets.DOCKER_PASSWORD }} + docker-remote-multi-platform: true + docker-arm-host: ${{ secrets.ARM_RUNNER_HOSTNAME }} + docker-arm-host-key: ${{ secrets.ARM_RUNNER_KEY }} + + - name: Build and push Docker image for Celo (indexer + API) + uses: docker/build-push-action@v5 + with: + context: . + file: ./docker/Dockerfile + push: true + tags: blockscout/blockscout-${{ env.DOCKER_CHAIN_NAME }}:${{ env.RELEASE_VERSION }}-postrelease-${{ env.SHORT_SHA }} + labels: ${{ steps.setup.outputs.docker-labels }} + platforms: | + linux/amd64 + linux/arm64/v8 + build-args: | + CACHE_EXCHANGE_RATES_PERIOD= + API_V1_READ_METHODS_DISABLED=false + DISABLE_WEBAPP=false + API_V1_WRITE_METHODS_DISABLED=false + CACHE_TOTAL_GAS_USAGE_COUNTER_ENABLED= + ADMIN_PANEL_ENABLED=false + CACHE_ADDRESS_WITH_BALANCES_UPDATE_INTERVAL= + BLOCKSCOUT_VERSION=v${{ env.RELEASE_VERSION }}-beta.+commit.${{ env.SHORT_SHA }} + RELEASE_VERSION=${{ env.RELEASE_VERSION }} + CHAIN_TYPE=${{ env.DOCKER_CHAIN_NAME }} diff --git a/apps/block_scout_web/lib/block_scout_web/chain.ex b/apps/block_scout_web/lib/block_scout_web/chain.ex index 5e652ce4a5..b6b4ddd1c3 100644 --- a/apps/block_scout_web/lib/block_scout_web/chain.ex +++ b/apps/block_scout_web/lib/block_scout_web/chain.ex @@ -14,14 +14,18 @@ defmodule BlockScoutWeb.Chain do string_to_transaction_hash: 1 ] + import Explorer.PagingOptions, + only: [ + default_paging_options: 0, + page_size: 0 + ] + import Explorer.Helper, only: [parse_integer: 1] alias BlockScoutWeb.PagingHelper alias Ecto.Association.NotLoaded - alias Explorer.Chain.UserOperation alias Explorer.Account.{TagAddress, TagTransaction, WatchlistAddress} alias Explorer.Chain.Beacon.Reader, as: BeaconReader - alias Explorer.Chain.Block.Reward alias Explorer.Chain.{ Address, @@ -29,6 +33,7 @@ defmodule BlockScoutWeb.Chain do Address.CurrentTokenBalance, Beacon.Blob, Block, + Block.Reward, Hash, InternalTransaction, Log, @@ -59,15 +64,11 @@ defmodule BlockScoutWeb.Chain do end end - @page_size 50 - @default_paging_options %PagingOptions{page_size: @page_size + 1} + @page_size page_size() + @default_paging_options default_paging_options() @address_hash_len 40 @full_hash_len 64 - def default_paging_options do - @default_paging_options - end - def current_filter(%{paging_options: paging_options} = params) do params |> Map.get("filter") diff --git a/apps/block_scout_web/lib/block_scout_web/channels/address_channel.ex b/apps/block_scout_web/lib/block_scout_web/channels/address_channel.ex index 4ea5943e0c..7fd6932a41 100644 --- a/apps/block_scout_web/lib/block_scout_web/channels/address_channel.ex +++ b/apps/block_scout_web/lib/block_scout_web/channels/address_channel.ex @@ -38,6 +38,23 @@ defmodule BlockScoutWeb.AddressChannel do @burn_address_hash burn_address_hash @current_token_balances_limit 50 + case Application.compile_env(:explorer, :chain_type) do + :celo -> + @chain_type_transaction_associations [ + :gas_token + ] + + _ -> + @chain_type_transaction_associations [] + end + + @transaction_associations [ + from_address: [:names, :smart_contract, :proxy_implementations], + to_address: [:names, :smart_contract, :proxy_implementations], + created_contract_address: [:names, :smart_contract, :proxy_implementations] + ] ++ + @chain_type_transaction_associations + def join("addresses:" <> address_hash, _params, socket) do {:ok, %{}, assign(socket, :address_hash, address_hash)} end @@ -335,13 +352,7 @@ defmodule BlockScoutWeb.AddressChannel do TransactionViewAPI.render("transactions.json", %{ transactions: transactions - |> Repo.preload([ - [ - from_address: [:names, :smart_contract, :proxy_implementations], - to_address: [:names, :smart_contract, :proxy_implementations], - created_contract_address: [:names, :smart_contract, :proxy_implementations] - ] - ]), + |> Repo.preload(@transaction_associations), conn: nil }) diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/address_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/address_controller.ex index dd6a721d6b..f5c2ef44f4 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/address_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/address_controller.ex @@ -30,17 +30,33 @@ defmodule BlockScoutWeb.API.V2.AddressController do alias Explorer.Chain.Address.Counters alias Explorer.Chain.Token.Instance + alias BlockScoutWeb.API.V2.CeloView + alias Explorer.Chain.Celo.ElectionReward, as: CeloElectionReward + alias Explorer.Chain.Celo.Reader, as: CeloReader + alias Indexer.Fetcher.OnDemand.CoinBalance, as: CoinBalanceOnDemand alias Indexer.Fetcher.OnDemand.ContractCode, as: ContractCodeOnDemand alias Indexer.Fetcher.OnDemand.TokenBalance, as: TokenBalanceOnDemand + case Application.compile_env(:explorer, :chain_type) do + :celo -> + @chain_type_transaction_necessity_by_association %{ + :gas_token => :optional + } + + _ -> + @chain_type_transaction_necessity_by_association %{} + end + @transaction_necessity_by_association [ - necessity_by_association: %{ - [created_contract_address: [:names, :smart_contract, :proxy_implementations]] => :optional, - [from_address: [:names, :smart_contract, :proxy_implementations]] => :optional, - [to_address: [:names, :smart_contract, :proxy_implementations]] => :optional, - :block => :optional - }, + necessity_by_association: + %{ + [created_contract_address: [:names, :smart_contract, :proxy_implementations]] => :optional, + [from_address: [:names, :smart_contract, :proxy_implementations]] => :optional, + [to_address: [:names, :smart_contract, :proxy_implementations]] => :optional, + :block => :optional + } + |> Map.merge(@chain_type_transaction_necessity_by_association), api?: true ] @@ -79,6 +95,26 @@ defmodule BlockScoutWeb.API.V2.AddressController do @api_true [api?: true] + @celo_election_rewards_options [ + necessity_by_association: %{ + [ + account_address: [ + :names, + :smart_contract, + :proxy_implementations + ] + ] => :optional, + [ + associated_account_address: [ + :names, + :smart_contract, + :proxy_implementations + ] + ] => :optional + }, + api?: true + ] + action_fallback(BlockScoutWeb.API.V2.FallbackController) def address(conn, %{"address_hash_param" => address_hash_string} = params) do @@ -447,20 +483,35 @@ defmodule BlockScoutWeb.API.V2.AddressController do def tabs_counters(conn, %{"address_hash_param" => address_hash_string} = params) do with {:ok, address_hash, _address} <- validate_address(address_hash_string, params) do - {validations, transactions, token_transfers, token_balances, logs, withdrawals, internal_txs} = - Counters.address_limited_counters(address_hash, @api_true) + counter_name_to_json_field_name = %{ + validations: :validations_count, + txs: :transactions_count, + token_transfers: :token_transfers_count, + token_balances: :token_balances_count, + logs: :logs_count, + withdrawals: :withdrawals_count, + internal_txs: :internal_txs_count, + celo_election_rewards: :celo_election_rewards_count + } + + counters_json = + address_hash + |> Counters.address_limited_counters(@api_true) + |> Enum.reduce(%{}, fn {counter_name, counter_value}, acc -> + counter_name_to_json_field_name + |> Map.fetch(counter_name) + |> case do + {:ok, json_field_name} -> + Map.put(acc, json_field_name, counter_value) + + :error -> + acc + end + end) conn |> put_status(200) - |> json(%{ - validations_count: validations, - transactions_count: transactions, - token_transfers_count: token_transfers, - token_balances_count: token_balances, - logs_count: logs, - withdrawals_count: withdrawals, - internal_txs_count: internal_txs - }) + |> json(counters_json) end end @@ -520,6 +571,37 @@ defmodule BlockScoutWeb.API.V2.AddressController do end end + @doc """ + Function to handle GET requests to `/api/v2/addresses/:address_hash_param/election-rewards` endpoint. + """ + def celo_election_rewards(conn, %{"address_hash_param" => address_hash_string} = params) do + with {:ok, address_hash, _address} <- validate_address(address_hash_string, params) do + full_options = + @celo_election_rewards_options + |> Keyword.merge(CeloElectionReward.address_paging_options(params)) + + results_plus_one = CeloReader.address_hash_to_election_rewards(address_hash, full_options) + + {rewards, next_page} = split_list_by_page(results_plus_one) + + next_page_params = + next_page_params( + next_page, + rewards, + delete_parameters_from_next_page_params(params), + &CeloElectionReward.to_address_paging_params/1 + ) + + conn + |> put_status(200) + |> put_view(CeloView) + |> render(:celo_election_rewards, %{ + rewards: rewards, + next_page_params: next_page_params + }) + end + end + @doc """ Checks if this valid address hash string, and this address is not prohibited address. Returns the `{:ok, address_hash, address}` if address hash passed all the checks. diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/advanced_filter_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/advanced_filter_controller.ex index a3c2f332a2..3178aaae5d 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/advanced_filter_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/advanced_filter_controller.ex @@ -1,7 +1,8 @@ defmodule BlockScoutWeb.API.V2.AdvancedFilterController do use BlockScoutWeb, :controller - import BlockScoutWeb.Chain, only: [default_paging_options: 0, split_list_by_page: 1, next_page_params: 4] + import BlockScoutWeb.Chain, only: [split_list_by_page: 1, next_page_params: 4] + import Explorer.PagingOptions, only: [default_paging_options: 0] alias BlockScoutWeb.API.V2.{AdvancedFilterView, CSVExportController, TransactionView} alias Explorer.{Chain, PagingOptions} diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/block_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/block_controller.ex index e4495740cd..4db96f9c11 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/block_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/block_controller.ex @@ -17,9 +17,23 @@ defmodule BlockScoutWeb.API.V2.BlockController do import Explorer.MicroserviceInterfaces.BENS, only: [maybe_preload_ens: 1] import Explorer.MicroserviceInterfaces.Metadata, only: [maybe_preload_metadata: 1] - alias BlockScoutWeb.API.V2.{TransactionView, WithdrawalView} + import Explorer.Chain.Celo.Helper, + only: [ + validate_epoch_block_number: 1, + block_number_to_epoch_number: 1 + ] + + alias BlockScoutWeb.API.V2.{ + CeloView, + TransactionView, + WithdrawalView + } + alias Explorer.Chain alias Explorer.Chain.Arbitrum.Reader, as: ArbitrumReader + alias Explorer.Chain.Celo.ElectionReward, as: CeloElectionReward + alias Explorer.Chain.Celo.EpochReward, as: CeloEpochReward + alias Explorer.Chain.Celo.Reader, as: CeloReader alias Explorer.Chain.InternalTransaction case Application.compile_env(:explorer, :chain_type) do @@ -46,6 +60,12 @@ defmodule BlockScoutWeb.API.V2.BlockController do :zksync_execute_transaction => :optional } + :celo -> + @chain_type_transaction_necessity_by_association %{ + :gas_token => :optional + } + @chain_type_block_necessity_by_association %{} + :arbitrum -> @chain_type_transaction_necessity_by_association %{} @chain_type_block_necessity_by_association %{ @@ -278,9 +298,123 @@ defmodule BlockScoutWeb.API.V2.BlockController do end end + @doc """ + Function to handle GET requests to `/api/v2/blocks/:block_hash_or_number/epoch` endpoint. + """ + @spec celo_epoch(Plug.Conn.t(), map()) :: + {:error, :not_found | {:invalid, :hash | :number | :celo_election_reward_type}} + | {:lost_consensus, {:error, :not_found} | {:ok, Explorer.Chain.Block.t()}} + | Plug.Conn.t() + def celo_epoch(conn, %{"block_hash_or_number" => block_hash_or_number}) do + params = [ + necessity_by_association: %{ + :celo_epoch_reward => :optional + }, + api?: true + ] + + with {:ok, block} <- block_param_to_block(block_hash_or_number, params), + :ok <- validate_epoch_block_number(block.number) do + epoch_number = block_number_to_epoch_number(block.number) + + epoch_distribution = + block + |> Map.get(:celo_epoch_reward) + |> case do + %CeloEpochReward{} = epoch_reward -> + CeloEpochReward.load_token_transfers(epoch_reward, api?: true) + + _ -> + nil + end + + aggregated_election_rewards = + CeloReader.block_hash_to_aggregated_election_rewards_by_type( + block.hash, + api?: true + ) + + conn + |> put_status(200) + |> put_view(CeloView) + |> render(:celo_epoch, %{ + epoch_number: epoch_number, + epoch_distribution: epoch_distribution, + aggregated_election_rewards: aggregated_election_rewards + }) + end + end + + @doc """ + Function to handle GET requests to `/api/v2/blocks/:block_hash_or_number/election-rewards/:reward_type` endpoint. + """ + @spec celo_election_rewards(Plug.Conn.t(), map()) :: + {:error, :not_found | {:invalid, :hash | :number | :celo_election_reward_type}} + | {:lost_consensus, {:error, :not_found} | {:ok, Explorer.Chain.Block.t()}} + | Plug.Conn.t() + def celo_election_rewards( + conn, + %{"block_hash_or_number" => block_hash_or_number, "reward_type" => reward_type} = params + ) do + with {:ok, reward_type_atom} <- celo_reward_type_to_atom(reward_type), + {:ok, block} <- + block_param_to_block(block_hash_or_number) do + address_associations = [:names, :smart_contract, :proxy_implementations] + + full_options = + [ + necessity_by_association: %{ + [account_address: address_associations] => :optional, + [associated_account_address: address_associations] => :optional + } + ] + |> Keyword.merge(CeloElectionReward.block_paging_options(params)) + |> Keyword.merge(@api_true) + + rewards_plus_one = + CeloReader.block_hash_to_election_rewards_by_type( + block.hash, + reward_type_atom, + full_options + ) + + {rewards, next_page} = split_list_by_page(rewards_plus_one) + + filtered_params = + params + |> delete_parameters_from_next_page_params() + |> Map.delete("reward_type") + + next_page_params = + next_page_params( + next_page, + rewards, + filtered_params, + &CeloElectionReward.to_block_paging_params/1 + ) + + conn + |> put_status(200) + |> put_view(CeloView) + |> render(:celo_election_rewards, %{ + rewards: rewards, + next_page_params: next_page_params + }) + end + end + defp block_param_to_block(block_hash_or_number, options \\ @api_true) do with {:ok, type, value} <- parse_block_hash_or_number_param(block_hash_or_number) do fetch_block(type, value, options) end end + + defp celo_reward_type_to_atom(reward_type_string) do + reward_type_string + |> CeloElectionReward.type_from_string() + |> case do + {:ok, type} -> {:ok, type} + :error -> {:error, {:invalid, :celo_election_reward_type}} + end + end end diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/fallback_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/fallback_controller.ex index 19448ad5fa..7f2544b453 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/fallback_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/fallback_controller.ex @@ -12,6 +12,7 @@ defmodule BlockScoutWeb.API.V2.FallbackController do @invalid_hash "Invalid hash" @invalid_number "Invalid number" @invalid_url "Invalid URL" + @invalid_celo_election_reward_type "Invalid Celo reward type, allowed types are: validator, group, voter, delegated-payment" @not_found "Not found" @contract_interaction_disabled "Contract interaction disabled" @restricted_access "Restricted access" @@ -97,26 +98,23 @@ defmodule BlockScoutWeb.API.V2.FallbackController do |> render(:message, %{message: @contract_interaction_disabled}) end - def call(conn, {:error, {:invalid, :hash}}) do - Logger.error(fn -> - ["#{@invalid_hash}"] - end) - - conn - |> put_status(:unprocessable_entity) - |> put_view(ApiView) - |> render(:message, %{message: @invalid_hash}) - end + def call(conn, {:error, {:invalid, entity}}) + when entity in ~w(hash number celo_election_reward_type)a do + message = + case entity do + :hash -> @invalid_hash + :number -> @invalid_number + :celo_election_reward_type -> @invalid_celo_election_reward_type + end - def call(conn, {:error, {:invalid, :number}}) do Logger.error(fn -> - ["#{@invalid_number}"] + ["#{message}"] end) conn |> put_status(:unprocessable_entity) |> put_view(ApiView) - |> render(:message, %{message: @invalid_number}) + |> render(:message, %{message: message}) end def call(conn, {:error, :not_found}) do diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/main_page_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/main_page_controller.ex index ecfaebd16f..79a89f8edd 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/main_page_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/main_page_controller.ex @@ -10,13 +10,25 @@ defmodule BlockScoutWeb.API.V2.MainPageController do import Explorer.MicroserviceInterfaces.BENS, only: [maybe_preload_ens: 1] import Explorer.MicroserviceInterfaces.Metadata, only: [maybe_preload_metadata: 1] + case Application.compile_env(:explorer, :chain_type) do + :celo -> + @chain_type_transaction_necessity_by_association %{ + :gas_token => :optional + } + + _ -> + @chain_type_transaction_necessity_by_association %{} + end + @transactions_options [ - necessity_by_association: %{ - :block => :required, - [created_contract_address: [:names, :smart_contract, :proxy_implementations]] => :optional, - [from_address: [:names, :smart_contract, :proxy_implementations]] => :optional, - [to_address: [:names, :smart_contract, :proxy_implementations]] => :optional - }, + necessity_by_association: + %{ + :block => :required, + [created_contract_address: [:names, :smart_contract, :proxy_implementations]] => :optional, + [from_address: [:names, :smart_contract, :proxy_implementations]] => :optional, + [to_address: [:names, :smart_contract, :proxy_implementations]] => :optional + } + |> Map.merge(@chain_type_transaction_necessity_by_association), paging_options: %PagingOptions{page_size: 6}, api?: true ] diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/mud_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/mud_controller.ex index 6cbba16d32..bf735edbaa 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/mud_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/mud_controller.ex @@ -4,11 +4,11 @@ defmodule BlockScoutWeb.API.V2.MudController do import BlockScoutWeb.Chain, only: [ next_page_params: 4, - split_list_by_page: 1, - default_paging_options: 0 + split_list_by_page: 1 ] import BlockScoutWeb.PagingHelper, only: [mud_records_sorting: 1] + import Explorer.PagingOptions, only: [default_paging_options: 0] import Explorer.MicroserviceInterfaces.BENS, only: [maybe_preload_ens: 1] import Explorer.MicroserviceInterfaces.Metadata, only: [maybe_preload_metadata: 1] diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/token_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/token_controller.ex index 86f9093d5f..934b4dbaf7 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/token_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/token_controller.ex @@ -17,8 +17,7 @@ defmodule BlockScoutWeb.API.V2.TokenController do next_page_params: 3, token_transfers_next_page_params: 3, unique_tokens_paging_options: 1, - unique_tokens_next_page: 3, - default_paging_options: 0 + unique_tokens_next_page: 3 ] import BlockScoutWeb.PagingHelper, @@ -31,6 +30,7 @@ defmodule BlockScoutWeb.API.V2.TokenController do import Explorer.MicroserviceInterfaces.BENS, only: [maybe_preload_ens: 1] import Explorer.MicroserviceInterfaces.Metadata, only: [maybe_preload_metadata: 1] + import Explorer.PagingOptions, only: [default_paging_options: 0] action_fallback(BlockScoutWeb.API.V2.FallbackController) diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/transaction_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/transaction_controller.ex index 485411b115..0ebb3a4c6b 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/transaction_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/transaction_controller.ex @@ -54,6 +54,11 @@ defmodule BlockScoutWeb.API.V2.TransactionController do :beacon_blob_transaction => :optional } + :celo -> + @chain_type_transaction_necessity_by_association %{ + :gas_token => :optional + } + _ -> @chain_type_transaction_necessity_by_association %{} end diff --git a/apps/block_scout_web/lib/block_scout_web/models/transaction_state_helper.ex b/apps/block_scout_web/lib/block_scout_web/models/transaction_state_helper.ex index 8d492ed184..3d96d69ff1 100644 --- a/apps/block_scout_web/lib/block_scout_web/models/transaction_state_helper.ex +++ b/apps/block_scout_web/lib/block_scout_web/models/transaction_state_helper.ex @@ -3,7 +3,7 @@ defmodule BlockScoutWeb.Models.TransactionStateHelper do Transaction state changes related functions """ - import BlockScoutWeb.Chain, only: [default_paging_options: 0] + import Explorer.PagingOptions, only: [default_paging_options: 0] import Explorer.Chain.SmartContract, only: [burn_address_hash_string: 0] alias Explorer.Chain.Transaction.StateChange diff --git a/apps/block_scout_web/lib/block_scout_web/routers/api_router.ex b/apps/block_scout_web/lib/block_scout_web/routers/api_router.ex index 3aee52a138..b976c38720 100644 --- a/apps/block_scout_web/lib/block_scout_web/routers/api_router.ex +++ b/apps/block_scout_web/lib/block_scout_web/routers/api_router.ex @@ -158,6 +158,11 @@ defmodule BlockScoutWeb.Routers.ApiRouter do if Application.compile_env(:explorer, :chain_type) == :arbitrum do get("/arbitrum-batch/:batch_number", V2.BlockController, :arbitrum_batch) end + + if Application.compile_env(:explorer, :chain_type) == :celo do + get("/:block_hash_or_number/epoch", V2.BlockController, :celo_epoch) + get("/:block_hash_or_number/election-rewards/:reward_type", V2.BlockController, :celo_election_rewards) + end end scope "/addresses" do @@ -177,6 +182,10 @@ defmodule BlockScoutWeb.Routers.ApiRouter do get("/:address_hash_param/withdrawals", V2.AddressController, :withdrawals) get("/:address_hash_param/nft", V2.AddressController, :nft_list) get("/:address_hash_param/nft/collections", V2.AddressController, :nft_collections) + + if Application.compile_env(:explorer, :chain_type) == :celo do + get("/:address_hash_param/election-rewards", V2.AddressController, :celo_election_rewards) + end end scope "/main-page" do diff --git a/apps/block_scout_web/lib/block_scout_web/views/api/v2/block_view.ex b/apps/block_scout_web/lib/block_scout_web/views/api/v2/block_view.ex index 0df80c9384..75a22551b8 100644 --- a/apps/block_scout_web/lib/block_scout_web/views/api/v2/block_view.ex +++ b/apps/block_scout_web/lib/block_scout_web/views/api/v2/block_view.ex @@ -146,6 +146,12 @@ defmodule BlockScoutWeb.API.V2.BlockView do BlockScoutWeb.API.V2.EthereumView.extend_block_json_response(result, block, single_block?) end + :celo -> + defp chain_type_fields(result, block, single_block?) do + # credo:disable-for-next-line Credo.Check.Design.AliasUsage + BlockScoutWeb.API.V2.CeloView.extend_block_json_response(result, block, single_block?) + end + _ -> defp chain_type_fields(result, _block, _single_block?) do result diff --git a/apps/block_scout_web/lib/block_scout_web/views/api/v2/celo_view.ex b/apps/block_scout_web/lib/block_scout_web/views/api/v2/celo_view.ex new file mode 100644 index 0000000000..eab95a7559 --- /dev/null +++ b/apps/block_scout_web/lib/block_scout_web/views/api/v2/celo_view.ex @@ -0,0 +1,391 @@ +defmodule BlockScoutWeb.API.V2.CeloView do + @moduledoc """ + View functions for rendering Celo-related data in JSON format. + """ + + require Logger + + import Explorer.Chain.SmartContract, only: [dead_address_hash_string: 0] + + alias BlockScoutWeb.API.V2.{Helper, TokenView, TransactionView} + alias Ecto.Association.NotLoaded + alias Explorer.Chain + alias Explorer.Chain.Cache.CeloCoreContracts + alias Explorer.Chain.Celo.Helper, as: CeloHelper + alias Explorer.Chain.Celo.{ElectionReward, EpochReward} + alias Explorer.Chain.Hash + alias Explorer.Chain.{Block, Transaction} + + @address_params [ + necessity_by_association: %{ + :names => :optional, + :smart_contract => :optional, + :proxy_implementations => :optional + }, + api?: true + ] + + def render("celo_epoch_distribution.json", %EpochReward{ + reserve_bolster_transfer: reserve_bolster_transfer, + community_transfer: community_transfer, + carbon_offsetting_transfer: carbon_offsetting_transfer + }) do + Map.new( + [ + reserve_bolster_transfer: reserve_bolster_transfer, + community_transfer: community_transfer, + carbon_offsetting_transfer: carbon_offsetting_transfer + ], + fn {field, token_transfer} -> + token_transfer_json = + token_transfer && + TransactionView.render( + "token_transfer.json", + %{token_transfer: token_transfer, conn: nil} + ) + + {field, token_transfer_json} + end + ) + end + + def render("celo_epoch_distribution.json", _distribution), + do: nil + + def render("celo_aggregated_election_rewards.json", aggregated_election_rewards) do + Map.new( + aggregated_election_rewards, + fn {type, %{total: total, count: count, token: token}} -> + {type, + %{ + total: total, + count: count, + token: + TokenView.render("token.json", %{ + token: token, + contract_address_hash: token.contract_address_hash + }) + }} + end + ) + end + + def render("celo_epoch.json", %{ + epoch_number: epoch_number, + epoch_distribution: epoch_distribution, + aggregated_election_rewards: aggregated_election_rewards + }) do + distribution_json = render("celo_epoch_distribution.json", epoch_distribution) + + # Workaround: we assume that if epoch rewards are not fetched for a block, + # we should not display aggregated election rewards for it. + # + # todo: consider checking pending block epoch operations to determine if + # epoch is fetched or not + aggregated_election_rewards_json = + if distribution_json do + render("celo_aggregated_election_rewards.json", aggregated_election_rewards) + else + nil + end + + %{ + number: epoch_number, + distribution: distribution_json, + aggregated_election_rewards: aggregated_election_rewards_json + } + end + + def render("celo_base_fee.json", %Block{} = block) do + # For the blocks, where both FeeHandler and Governance contracts aren't + # deployed, the base fee is not burnt, but refunded to transaction sender, + # so we return nil in this case. + + base_fee = Block.burnt_fees(block.transactions, block.base_fee_per_gas) + + fee_handler_base_fee_breakdown( + base_fee, + block.number + ) || + governance_base_fee_breakdown( + base_fee, + block.number + ) + end + + def render("celo_election_rewards.json", %{ + rewards: rewards, + next_page_params: next_page_params + }) do + %{ + "items" => Enum.map(rewards, &prepare_election_reward/1), + "next_page_params" => next_page_params + } + end + + @doc """ + Extends the JSON output with a sub-map containing information related to Celo, + such as the epoch number, whether the block is an epoch block, and the routing + of the base fee. + + ## Parameters + - `out_json`: A map defining the output JSON which will be extended. + - `block`: The block structure containing Celo-related data. + - `single_block?`: A boolean indicating if it is a single block. + + ## Returns + - A map extended with data related to Celo. + """ + def extend_block_json_response(out_json, %Block{} = block, single_block?) do + celo_json = + %{ + "is_epoch_block" => CeloHelper.epoch_block_number?(block.number), + "epoch_number" => CeloHelper.block_number_to_epoch_number(block.number) + } + |> maybe_add_base_fee_info(block, single_block?) + + Map.put(out_json, "celo", celo_json) + end + + @doc """ + Extends the JSON output with a sub-map containing information about the gas + token used to pay for the transaction fees. + + ## Parameters + - `out_json`: A map defining the output JSON which will be extended. + - `transaction`: The transaction structure containing Celo-related data. + + ## Returns + - A map extended with data related to the gas token. + """ + def extend_transaction_json_response(out_json, %Transaction{} = transaction) do + token_json = + case { + Map.get(transaction, :gas_token_contract_address), + Map.get(transaction, :gas_token) + } do + # {_, %NotLoaded{}} -> + # nil + + {nil, _} -> + nil + + {gas_token_contract_address, gas_token} -> + if is_nil(gas_token) do + Logger.error(fn -> + [ + "Transaction #{transaction.hash} has a ", + "gas token contract address #{gas_token_contract_address} ", + "but no associated token found in the database" + ] + end) + end + + TokenView.render("token.json", %{ + token: gas_token, + contract_address_hash: gas_token_contract_address + }) + end + + Map.put(out_json, "celo", %{"gas_token" => token_json}) + end + + @spec prepare_election_reward(Explorer.Chain.Celo.ElectionReward.t()) :: %{ + :account => nil | %{optional(String.t()) => any()}, + :amount => Decimal.t(), + :associated_account => nil | %{optional(String.t()) => any()}, + optional(:block_hash) => Hash.Full.t(), + optional(:block_number) => Block.block_number(), + optional(:epoch_number) => non_neg_integer(), + optional(:type) => ElectionReward.type() + } + defp prepare_election_reward(%ElectionReward{block: %NotLoaded{}} = reward) do + %{ + amount: reward.amount, + account: + Helper.address_with_info( + reward.account_address, + reward.account_address_hash + ), + associated_account: + Helper.address_with_info( + reward.associated_account_address, + reward.associated_account_address_hash + ) + } + end + + defp prepare_election_reward(%ElectionReward{} = reward) do + %{ + amount: reward.amount, + block_number: reward.block.number, + block_hash: reward.block_hash, + epoch_number: reward.block.number |> CeloHelper.block_number_to_epoch_number(), + account: + Helper.address_with_info( + reward.account_address, + reward.account_address_hash + ), + associated_account: + Helper.address_with_info( + reward.associated_account_address, + reward.associated_account_address_hash + ), + type: reward.type + } + end + + # Convert the burn fraction from FixidityLib value to decimal. + @spec burn_fraction_decimal(integer()) :: Decimal.t() + defp burn_fraction_decimal(burn_fraction_fixidity_lib) + when is_integer(burn_fraction_fixidity_lib) do + base = Decimal.new(1, 10, 24) + fraction = Decimal.new(1, burn_fraction_fixidity_lib, 0) + Decimal.div(fraction, base) + end + + # Get the breakdown of the base fee for the case when FeeHandler is a contract + # that receives the base fee. + @spec fee_handler_base_fee_breakdown(Decimal.t(), Block.block_number()) :: + %{ + :recipient => %{optional(String.t()) => any()}, + :amount => float(), + :breakdown => [ + %{ + :address => %{optional(String.t()) => any()}, + :amount => float(), + :percentage => float() + } + ] + } + | nil + defp fee_handler_base_fee_breakdown(base_fee, block_number) do + with {:ok, fee_handler_contract_address_hash} <- + CeloCoreContracts.get_address(:fee_handler, block_number), + {:ok, %{"address" => fee_beneficiary_address_hash}} <- + CeloCoreContracts.get_event(:fee_handler, :fee_beneficiary_set, block_number), + {:ok, %{"value" => burn_fraction_fixidity_lib}} <- + CeloCoreContracts.get_event(:fee_handler, :burn_fraction_set, block_number) do + burn_fraction = burn_fraction_decimal(burn_fraction_fixidity_lib) + + burnt_amount = Decimal.mult(base_fee, burn_fraction) + burnt_percentage = Decimal.mult(burn_fraction, 100) + + carbon_offsetting_amount = Decimal.sub(base_fee, burnt_amount) + carbon_offsetting_percentage = Decimal.sub(100, burnt_percentage) + + celo_burn_address_hash_string = dead_address_hash_string() + + address_hashes_to_fetch_from_db = [ + fee_handler_contract_address_hash, + fee_beneficiary_address_hash, + celo_burn_address_hash_string + ] + + address_hash_string_to_address = + address_hashes_to_fetch_from_db + |> Enum.map(&(&1 |> Chain.string_to_address_hash() |> elem(1))) + # todo: Querying database in the view is not a good practice. Consider + # refactoring. + |> Chain.hashes_to_addresses(@address_params) + |> Map.new(fn address -> + { + to_string(address.hash), + address + } + end) + + %{ + ^fee_handler_contract_address_hash => fee_handler_contract_address_info, + ^fee_beneficiary_address_hash => fee_beneficiary_address_info, + ^celo_burn_address_hash_string => burn_address_info + } = + Map.new( + address_hashes_to_fetch_from_db, + &{ + &1, + Helper.address_with_info( + Map.get(address_hash_string_to_address, &1), + &1 + ) + } + ) + + %{ + recipient: fee_handler_contract_address_info, + amount: base_fee, + breakdown: [ + %{ + address: burn_address_info, + amount: Decimal.to_float(burnt_amount), + percentage: Decimal.to_float(burnt_percentage) + }, + %{ + address: fee_beneficiary_address_info, + amount: Decimal.to_float(carbon_offsetting_amount), + percentage: Decimal.to_float(carbon_offsetting_percentage) + } + ] + } + else + _ -> nil + end + end + + # Get the breakdown of the base fee for the case when Governance is a contract + # that receives the base fee. + # + # Note that the base fee is not burnt in this case, but simply kept on the + # contract balance. + @spec governance_base_fee_breakdown(Decimal.t(), Block.block_number()) :: + %{ + :recipient => %{optional(String.t()) => any()}, + :amount => float(), + :breakdown => [ + %{ + :address => %{optional(String.t()) => any()}, + :amount => float(), + :percentage => float() + } + ] + } + | nil + defp governance_base_fee_breakdown(base_fee, block_number) do + with {:ok, address_hash_string} when not is_nil(address_hash_string) <- + CeloCoreContracts.get_address(:governance, block_number), + {:ok, address_hash} <- Chain.string_to_address_hash(address_hash_string) do + address = + address_hash + # todo: Querying database in the view is not a good practice. Consider + # refactoring. + |> Chain.hash_to_address(@address_params) + |> case do + {:ok, address} -> address + {:error, :not_found} -> nil + end + + address_with_info = + Helper.address_with_info( + address, + address_hash + ) + + %{ + recipient: address_with_info, + amount: base_fee, + breakdown: [] + } + else + _ -> + nil + end + end + + defp maybe_add_base_fee_info(celo_json, block_or_transaction, true) do + base_fee_breakdown_json = render("celo_base_fee.json", block_or_transaction) + Map.put(celo_json, "base_fee", base_fee_breakdown_json) + end + + defp maybe_add_base_fee_info(celo_json, _block_or_transaction, false), + do: celo_json +end diff --git a/apps/block_scout_web/lib/block_scout_web/views/api/v2/transaction_view.ex b/apps/block_scout_web/lib/block_scout_web/views/api/v2/transaction_view.ex index 8198573176..0b43013a4a 100644 --- a/apps/block_scout_web/lib/block_scout_web/views/api/v2/transaction_view.ex +++ b/apps/block_scout_web/lib/block_scout_web/views/api/v2/transaction_view.ex @@ -909,6 +909,16 @@ defmodule BlockScoutWeb.API.V2.TransactionView do BlockScoutWeb.API.V2.EthereumView.extend_transaction_json_response(result, transaction) end + :celo -> + defp chain_type_transformations(transactions) do + transactions + end + + defp chain_type_fields(result, transaction, _single_tx?, _conn, _watchlist_names) do + # credo:disable-for-next-line Credo.Check.Design.AliasUsage + BlockScoutWeb.API.V2.CeloView.extend_transaction_json_response(result, transaction) + end + _ -> defp chain_type_transformations(transactions) do transactions diff --git a/apps/block_scout_web/test/block_scout_web/controllers/api/v2/block_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/api/v2/block_controller_test.exs index 9ba0952cb1..bfb690c04d 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/api/v2/block_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/api/v2/block_controller_test.exs @@ -9,6 +9,28 @@ defmodule BlockScoutWeb.API.V2.BlockControllerTest do Supervisor.terminate_child(Explorer.Supervisor, Explorer.Chain.Cache.Uncles.child_id()) Supervisor.restart_child(Explorer.Supervisor, Explorer.Chain.Cache.Uncles.child_id()) + Application.put_env(:explorer, Explorer.Chain.Cache.CeloCoreContracts, + contracts: %{ + "addresses" => %{ + "Accounts" => [], + "Election" => [], + "EpochRewards" => [], + "FeeHandler" => [], + "GasPriceMinimum" => [], + "GoldToken" => [], + "Governance" => [], + "LockedGold" => [], + "Reserve" => [], + "StableToken" => [], + "Validators" => [] + } + } + ) + + on_exit(fn -> + Application.put_env(:explorer, Explorer.Chain.Cache.CeloCoreContracts, contracts: %{}) + end) + :ok end diff --git a/apps/block_scout_web/test/block_scout_web/controllers/api/v2/transaction_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/api/v2/transaction_controller_test.exs index f28b729e29..148d172265 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/api/v2/transaction_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/api/v2/transaction_controller_test.exs @@ -1103,6 +1103,185 @@ defmodule BlockScoutWeb.API.V2.TransactionControllerTest do end end + if Application.compile_env(:explorer, :chain_type) == :celo do + describe "celo gas token" do + test "when gas is paid with token and token is present in db", %{conn: conn} do + token = insert(:token) + + tx = + :transaction + |> insert(gas_token_contract_address: token.contract_address) + |> with_block() + + request = get(conn, "/api/v2/transactions") + + token_address_hash = Address.checksum(token.contract_address_hash) + token_type = token.type + token_name = token.name + token_symbol = token.symbol + + assert %{ + "items" => [ + %{ + "celo" => %{ + "gas_token" => %{ + "address" => ^token_address_hash, + "name" => ^token_name, + "symbol" => ^token_symbol, + "type" => ^token_type + } + } + } + ] + } = json_response(request, 200) + + request = get(conn, "/api/v2/transactions/#{to_string(tx.hash)}") + + assert %{ + "celo" => %{ + "gas_token" => %{ + "address" => ^token_address_hash, + "name" => ^token_name, + "symbol" => ^token_symbol, + "type" => ^token_type + } + } + } = json_response(request, 200) + + request = get(conn, "/api/v2/addresses/#{to_string(tx.from_address_hash)}/transactions") + + assert %{ + "items" => [ + %{ + "celo" => %{ + "gas_token" => %{ + "address" => ^token_address_hash, + "name" => ^token_name, + "symbol" => ^token_symbol, + "type" => ^token_type + } + } + } + ] + } = json_response(request, 200) + + request = get(conn, "/api/v2/main-page/transactions") + + assert [ + %{ + "celo" => %{ + "gas_token" => %{ + "address" => ^token_address_hash, + "name" => ^token_name, + "symbol" => ^token_symbol, + "type" => ^token_type + } + } + } + ] = json_response(request, 200) + end + + test "when gas is paid with token and token is not present in db", %{conn: conn} do + unknown_token_address = insert(:address) + + tx = + :transaction + |> insert(gas_token_contract_address: unknown_token_address) + |> with_block() + + unknown_token_address_hash = Address.checksum(unknown_token_address.hash) + + request = get(conn, "/api/v2/transactions") + + assert %{ + "items" => [ + %{ + "celo" => %{ + "gas_token" => %{ + "address" => ^unknown_token_address_hash + } + } + } + ] + } = json_response(request, 200) + + request = get(conn, "/api/v2/transactions/#{to_string(tx.hash)}") + + assert %{ + "celo" => %{ + "gas_token" => %{ + "address" => ^unknown_token_address_hash + } + } + } = json_response(request, 200) + + request = get(conn, "/api/v2/addresses/#{to_string(tx.from_address_hash)}/transactions") + + assert %{ + "items" => [ + %{ + "celo" => %{ + "gas_token" => %{ + "address" => ^unknown_token_address_hash + } + } + } + ] + } = json_response(request, 200) + + request = get(conn, "/api/v2/main-page/transactions") + + assert [ + %{ + "celo" => %{ + "gas_token" => %{ + "address" => ^unknown_token_address_hash + } + } + } + ] = json_response(request, 200) + end + + test "when gas is paid in native coin", %{conn: conn} do + tx = :transaction |> insert() |> with_block() + + request = get(conn, "/api/v2/transactions") + + assert %{ + "items" => [ + %{ + "celo" => %{"gas_token" => nil} + } + ] + } = json_response(request, 200) + + request = get(conn, "/api/v2/transactions/#{to_string(tx.hash)}") + + assert %{ + "celo" => %{"gas_token" => nil} + } = json_response(request, 200) + + request = get(conn, "/api/v2/addresses/#{to_string(tx.from_address_hash)}/transactions") + + assert %{ + "items" => [ + %{ + "celo" => %{"gas_token" => nil} + } + ] + } = json_response(request, 200) + + request = get(conn, "/api/v2/main-page/transactions") + + assert [ + %{ + "celo" => %{"gas_token" => nil} + } + ] = json_response(request, 200) + end + end + end + if Application.compile_env(:explorer, :chain_type) == :stability do @first_topic_hex_string_1 "0x99e7b0ba56da2819c37c047f0511fd2bf6c9b4e27b4a979a19d6da0f74be8155" diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc.ex index 7cd5c64842..e960e08cf8 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc.ex @@ -579,4 +579,16 @@ defmodule EthereumJSONRPC do defp chunk_requests(requests, nil), do: requests defp chunk_requests(requests, chunk_size), do: Enum.chunk_every(requests, chunk_size) + + def put_if_present(result, map, keys) do + Enum.reduce(keys, result, fn {from_key, to_key}, acc -> + value = map[from_key] + + if value do + Map.put(acc, to_key, value) + else + acc + end + end) + end end diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/block.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/block.ex index 1c6b870b28..9b38692560 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/block.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/block.ex @@ -8,6 +8,11 @@ defmodule EthereumJSONRPC.Block do alias EthereumJSONRPC.{Transactions, Uncles, Withdrawals} + # Because proof of stake does not naturally produce uncles like proof of work, + # the list of these in each block is empty, and the hash of this list + # (sha3Uncles) is the RLP-encoded hash of an empty list. + @sha3_uncles_empty_list "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347" + case Application.compile_env(:explorer, :chain_type) do :rsk -> @chain_type_fields quote( @@ -320,13 +325,11 @@ defmodule EthereumJSONRPC.Block do "number" => number, "parentHash" => parent_hash, "receiptsRoot" => receipts_root, - "sha3Uncles" => sha3_uncles, "size" => size, "stateRoot" => state_root, "timestamp" => timestamp, "totalDifficulty" => total_difficulty, "transactionsRoot" => transactions_root, - "uncles" => uncles, "baseFeePerGas" => base_fee_per_gas } = elixir ) do @@ -343,13 +346,15 @@ defmodule EthereumJSONRPC.Block do number: number, parent_hash: parent_hash, receipts_root: receipts_root, - sha3_uncles: sha3_uncles, + # In case of CELO, `sha3_uncles` may not be returned by eth_getBlockByHash + sha3_uncles: Map.get(elixir, "sha3Uncles", @sha3_uncles_empty_list), size: size, state_root: state_root, timestamp: timestamp, total_difficulty: total_difficulty, transactions_root: transactions_root, - uncles: uncles, + # In case of CELO, `uncles` may not be returned by eth_getBlockByHash + uncles: Map.get(elixir, "uncles", []), base_fee_per_gas: base_fee_per_gas } end @@ -366,12 +371,10 @@ defmodule EthereumJSONRPC.Block do "number" => number, "parentHash" => parent_hash, "receiptsRoot" => receipts_root, - "sha3Uncles" => sha3_uncles, "size" => size, "stateRoot" => state_root, "timestamp" => timestamp, "transactionsRoot" => transactions_root, - "uncles" => uncles, "baseFeePerGas" => base_fee_per_gas } = elixir ) do @@ -388,12 +391,14 @@ defmodule EthereumJSONRPC.Block do number: number, parent_hash: parent_hash, receipts_root: receipts_root, - sha3_uncles: sha3_uncles, + # In case of CELO, `sha3_uncles` may not be returned by eth_getBlockByHash + sha3_uncles: Map.get(elixir, "sha3Uncles", @sha3_uncles_empty_list), size: size, state_root: state_root, timestamp: timestamp, transactions_root: transactions_root, - uncles: uncles, + # In case of CELO, `uncles` may not be returned by eth_getBlockByHash + uncles: Map.get(elixir, "uncles", []), base_fee_per_gas: base_fee_per_gas } end @@ -410,13 +415,11 @@ defmodule EthereumJSONRPC.Block do "number" => number, "parentHash" => parent_hash, "receiptsRoot" => receipts_root, - "sha3Uncles" => sha3_uncles, "size" => size, "stateRoot" => state_root, "timestamp" => timestamp, "totalDifficulty" => total_difficulty, - "transactionsRoot" => transactions_root, - "uncles" => uncles + "transactionsRoot" => transactions_root } = elixir ) do %{ @@ -432,13 +435,15 @@ defmodule EthereumJSONRPC.Block do number: number, parent_hash: parent_hash, receipts_root: receipts_root, - sha3_uncles: sha3_uncles, + # In case of CELO, `sha3_uncles` may not be returned by eth_getBlockByHash + sha3_uncles: Map.get(elixir, "sha3Uncles", @sha3_uncles_empty_list), size: size, state_root: state_root, timestamp: timestamp, total_difficulty: total_difficulty, transactions_root: transactions_root, - uncles: uncles + # In case of CELO, `uncles` may not be returned by eth_getBlockByHash + uncles: Map.get(elixir, "uncles", []) } end @@ -455,12 +460,10 @@ defmodule EthereumJSONRPC.Block do "number" => number, "parentHash" => parent_hash, "receiptsRoot" => receipts_root, - "sha3Uncles" => sha3_uncles, "size" => size, "stateRoot" => state_root, "timestamp" => timestamp, - "transactionsRoot" => transactions_root, - "uncles" => uncles + "transactionsRoot" => transactions_root } = elixir ) do %{ @@ -476,12 +479,14 @@ defmodule EthereumJSONRPC.Block do number: number, parent_hash: parent_hash, receipts_root: receipts_root, - sha3_uncles: sha3_uncles, + # In case of CELO, `sha3_uncles` may not be returned by eth_getBlockByHash + sha3_uncles: Map.get(elixir, "sha3Uncles", @sha3_uncles_empty_list), size: size, state_root: state_root, timestamp: timestamp, transactions_root: transactions_root, - uncles: uncles + # In case of CELO, `uncles` may not be returned by eth_getBlockByHash + uncles: Map.get(elixir, "uncles", []) } end @@ -659,6 +664,8 @@ defmodule EthereumJSONRPC.Block do |> Enum.map(fn {uncle_hash, index} -> %{"hash" => uncle_hash, "nephewHash" => nephew_hash, "index" => index} end) end + def elixir_to_uncles(_), do: [] + @doc """ Get `t:EthereumJSONRPC.Withdrawals.elixir/0` from `t:elixir/0`. diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/geth/call.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/geth/call.ex index d019e0210e..2d8e2723b5 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/geth/call.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/geth/call.ex @@ -3,8 +3,7 @@ defmodule EthereumJSONRPC.Geth.Call do A single call returned from [debug_traceTransaction](https://github.com/ethereum/go-ethereum/wiki/Management-APIs#debug_tracetransaction) using a custom tracer (`priv/js/ethereum_jsonrpc/geth/debug_traceTransaction/tracer.js`). """ - import EthereumJSONRPC, only: [quantity_to_integer: 1] - import EthereumJSONRPC.Transaction, only: [put_if_present: 3] + import EthereumJSONRPC, only: [quantity_to_integer: 1, put_if_present: 3] @doc """ A call can call another another contract: diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/logs.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/logs.ex index e62765d859..de6b81dbf3 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/logs.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/logs.ex @@ -4,7 +4,13 @@ defmodule EthereumJSONRPC.Logs do [`eth_getTransactionReceipt`](https://github.com/ethereum/wiki/wiki/JSON-RPC/e8e0771b9f3677693649d945956bc60e886ceb2b#eth_gettransactionreceipt). """ - alias EthereumJSONRPC.Log + import EthereumJSONRPC, + only: [ + integer_to_quantity: 1, + put_if_present: 3 + ] + + alias EthereumJSONRPC.{Log, Transport} @type elixir :: [Log.elixir()] @type t :: [Log.t()] @@ -18,4 +24,102 @@ defmodule EthereumJSONRPC.Logs do def to_elixir(logs) when is_list(logs) do Enum.map(logs, &Log.to_elixir/1) end + + @spec request( + id :: integer(), + params :: + %{ + :from_block => EthereumJSONRPC.tag() | EthereumJSONRPC.block_number(), + :to_block => EthereumJSONRPC.tag() | EthereumJSONRPC.block_number(), + optional(:topics) => list(EthereumJSONRPC.hash()), + optional(:address) => EthereumJSONRPC.address() + } + | %{ + :block_hash => EthereumJSONRPC.hash(), + optional(:topics) => list(EthereumJSONRPC.hash()), + optional(:address) => EthereumJSONRPC.address() + } + ) :: Transport.request() + def request(id, params) when is_integer(id) do + EthereumJSONRPC.request(%{ + id: id, + method: "eth_getLogs", + params: [to_request_params(params)] + }) + end + + defp to_request_params( + %{ + from_block: from_block, + to_block: to_block + } = params + ) do + %{ + fromBlock: block_number_to_quantity_or_tag(from_block), + toBlock: block_number_to_quantity_or_tag(to_block) + } + |> maybe_add_topics_and_address(params) + end + + defp to_request_params(%{block_hash: block_hash} = params) + when is_binary(block_hash) do + %{ + blockHash: block_hash + } + |> maybe_add_topics_and_address(params) + end + + defp maybe_add_topics_and_address(request_params, params) do + put_if_present(request_params, params, [ + {:topics, :topics}, + {:address, :address} + ]) + end + + defp block_number_to_quantity_or_tag(block_number) when is_integer(block_number) do + integer_to_quantity(block_number) + end + + defp block_number_to_quantity_or_tag(tag) when tag in ~w(earliest latest pending safe) do + tag + end + + def from_responses(responses) when is_list(responses) do + responses + |> reduce_responses() + |> case do + {:ok, logs} -> + { + :ok, + logs + |> to_elixir() + |> elixir_to_params() + } + + {:error, reasons} -> + {:error, reasons} + end + end + + defp reduce_responses(responses) do + responses + |> Enum.reduce( + {:ok, []}, + fn + %{result: result}, {:ok, logs} + when is_list(result) -> + {:ok, result ++ logs} + + %{result: _}, {:error, _} = error -> + error + + %{error: reason}, {:ok, _} -> + {:error, [reason]} + + %{error: reason}, {:error, reasons} + when is_list(reasons) -> + {:error, [reason | reasons]} + end + ) + end end diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/nethermind/trace.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/nethermind/trace.ex index 6e8cf42af8..17b781224c 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/nethermind/trace.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/nethermind/trace.ex @@ -4,7 +4,7 @@ defmodule EthereumJSONRPC.Nethermind.Trace do [`trace_replayTransaction`](https://openethereum.github.io/JSONRPC-trace-module#trace_replaytransaction). """ - import EthereumJSONRPC.Transaction, only: [put_if_present: 3] + import EthereumJSONRPC, only: [put_if_present: 3] alias EthereumJSONRPC.Nethermind.Trace.{Action, Result} @doc """ diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/transaction.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/transaction.ex index 8fb447894e..f835c4fb6f 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/transaction.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/transaction.ex @@ -7,7 +7,13 @@ defmodule EthereumJSONRPC.Transaction do [`eth_getTransactionByBlockHashAndIndex`](https://github.com/ethereum/wiki/wiki/JSON-RPC/e8e0771b9f3677693649d945956bc60e886ceb2b#eth_gettransactionbyblockhashandindex), and [`eth_getTransactionByBlockNumberAndIndex`](https://github.com/ethereum/wiki/wiki/JSON-RPC/e8e0771b9f3677693649d945956bc60e886ceb2b#eth_gettransactionbyblocknumberandindex) """ - import EthereumJSONRPC, only: [quantity_to_integer: 1, integer_to_quantity: 1, request: 1] + import EthereumJSONRPC, + only: [ + quantity_to_integer: 1, + integer_to_quantity: 1, + request: 1, + put_if_present: 3 + ] alias EthereumJSONRPC @@ -48,6 +54,15 @@ defmodule EthereumJSONRPC.Transaction do ] ) + :celo -> + @chain_type_fields quote( + do: [ + gas_token_contract_address_hash: EthereumJSONRPC.address(), + gas_fee_recipient_address_hash: EthereumJSONRPC.address(), + gateway_fee: non_neg_integer() + ] + ) + :arbitrum -> @chain_type_fields quote( do: [ @@ -103,6 +118,11 @@ defmodule EthereumJSONRPC.Transaction do * `"executionNode"` - `t:EthereumJSONRPC.address/0` of execution node (used by Suave). * `"requestRecord"` - map of wrapped transaction data (used by Suave). """ + :celo -> """ + * `"feeCurrency"` - `t:EthereumJSONRPC.address/0` of the currency used to pay for gas. + * `"gatewayFee"` - `t:EthereumJSONRPC.quantity/0` of the gateway fee. + * `"gatewayFeeRecipient"` - `t:EthereumJSONRPC.address/0` of the gateway fee recipient. + """ _ -> "" end} """ @@ -134,7 +154,7 @@ defmodule EthereumJSONRPC.Transaction do } @doc """ - Geth `elixir` can be converted to `params`. Geth does not supply `"publicKey"` or `"standardV"`, unlike Nethermind. + Geth `elixir` can be converted to `params`. Geth does not supply `"publicKey"` or `"standardV"`, unlike Nethermind. iex> EthereumJSONRPC.Transaction.elixir_to_params( ...> %{ @@ -516,6 +536,13 @@ defmodule EthereumJSONRPC.Transaction do }) end + :celo -> + put_if_present(params, elixir, [ + {"feeCurrency", :gas_token_contract_address_hash}, + {"gatewayFee", :gateway_fee}, + {"gatewayFeeRecipient", :gas_fee_recipient_address_hash} + ]) + :arbitrum -> put_if_present(params, elixir, [ {"requestId", :request_id} @@ -673,19 +700,17 @@ defmodule EthereumJSONRPC.Transaction do end end - defp entry_to_elixir(_) do - {:ignore, :ignore} - end + # Celo-specific fields + if Application.compile_env(:explorer, :chain_type) == :celo do + defp entry_to_elixir({key, value}) + when key in ~w(feeCurrency gatewayFeeRecipient), + do: {key, value} - def put_if_present(result, transaction, keys) do - Enum.reduce(keys, result, fn {from_key, to_key}, acc -> - value = transaction[from_key] + defp entry_to_elixir({"gatewayFee" = key, quantity_or_nil}), + do: {key, quantity_or_nil && quantity_to_integer(quantity_or_nil)} + end - if value do - Map.put(acc, to_key, value) - else - acc - end - end) + defp entry_to_elixir(_) do + {:ignore, :ignore} end end diff --git a/apps/explorer/config/dev.exs b/apps/explorer/config/dev.exs index 4fbc76dbe8..47e5689bff 100644 --- a/apps/explorer/config/dev.exs +++ b/apps/explorer/config/dev.exs @@ -23,6 +23,8 @@ config :explorer, Explorer.Repo.PolygonZkevm, timeout: :timer.seconds(80) # Configure ZkSync database config :explorer, Explorer.Repo.ZkSync, timeout: :timer.seconds(80) +config :explorer, Explorer.Repo.Celo, timeout: :timer.seconds(80) + config :explorer, Explorer.Repo.RSK, timeout: :timer.seconds(80) config :explorer, Explorer.Repo.Shibarium, timeout: :timer.seconds(80) diff --git a/apps/explorer/config/prod.exs b/apps/explorer/config/prod.exs index a7166e19aa..7af6ee2169 100644 --- a/apps/explorer/config/prod.exs +++ b/apps/explorer/config/prod.exs @@ -39,6 +39,11 @@ config :explorer, Explorer.Repo.ZkSync, timeout: :timer.seconds(60), ssl_opts: [verify: :verify_none] +config :explorer, Explorer.Repo.Celo, + prepare: :unnamed, + timeout: :timer.seconds(60), + ssl_opts: [verify: :verify_none] + config :explorer, Explorer.Repo.RSK, prepare: :unnamed, timeout: :timer.seconds(60), diff --git a/apps/explorer/config/test.exs b/apps/explorer/config/test.exs index b006aef935..6687e8df4d 100644 --- a/apps/explorer/config/test.exs +++ b/apps/explorer/config/test.exs @@ -59,6 +59,7 @@ for repo <- [ Explorer.Repo.PolygonEdge, Explorer.Repo.PolygonZkevm, Explorer.Repo.ZkSync, + Explorer.Repo.Celo, Explorer.Repo.RSK, Explorer.Repo.Shibarium, Explorer.Repo.Suave, diff --git a/apps/explorer/lib/explorer/account/notifier/forbidden_address.ex b/apps/explorer/lib/explorer/account/notifier/forbidden_address.ex index f977b3e4dc..16ab861c46 100644 --- a/apps/explorer/lib/explorer/account/notifier/forbidden_address.ex +++ b/apps/explorer/lib/explorer/account/notifier/forbidden_address.ex @@ -3,13 +3,17 @@ defmodule Explorer.Account.Notifier.ForbiddenAddress do Check if address is forbidden to notify """ - import Explorer.Chain.SmartContract, only: [burn_address_hash_string: 0] + import Explorer.Chain.SmartContract, + only: [ + burn_address_hash_string: 0, + dead_address_hash_string: 0 + ] alias Explorer.Chain.Address @blacklist [ burn_address_hash_string(), - "0x000000000000000000000000000000000000dEaD" + dead_address_hash_string() ] alias Explorer.AccessHelper diff --git a/apps/explorer/lib/explorer/application.ex b/apps/explorer/lib/explorer/application.ex index a92c10a8c1..59951fae2c 100644 --- a/apps/explorer/lib/explorer/application.ex +++ b/apps/explorer/lib/explorer/application.ex @@ -157,6 +157,7 @@ defmodule Explorer.Application do Explorer.Repo.PolygonEdge, Explorer.Repo.PolygonZkevm, Explorer.Repo.ZkSync, + Explorer.Repo.Celo, Explorer.Repo.RSK, Explorer.Repo.Shibarium, Explorer.Repo.Suave, diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index 7612ca8adc..6ce7ec66df 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -93,7 +93,8 @@ defmodule Explorer.Chain do alias Dataloader.Ecto, as: DataloaderEcto - @default_paging_options %PagingOptions{page_size: 50} + @default_page_size 50 + @default_paging_options %PagingOptions{page_size: @default_page_size} @token_transfers_per_transaction_preview 10 @token_transfers_necessity_by_association %{ @@ -121,7 +122,6 @@ defmodule Explorer.Chain do @revert_msg_prefix_6_empty "execution reverted" @limit_showing_transactions 10_000 - @default_page_size 50 @typedoc """ The name of an association on the `t:Ecto.Schema.t/0` diff --git a/apps/explorer/lib/explorer/chain/address/counters.ex b/apps/explorer/lib/explorer/chain/address/counters.ex index d6d1da965c..d58a55ed73 100644 --- a/apps/explorer/lib/explorer/chain/address/counters.ex +++ b/apps/explorer/lib/explorer/chain/address/counters.ex @@ -31,6 +31,7 @@ defmodule Explorer.Chain.Address.Counters do alias Explorer.Chain.Cache.AddressesTabsCounters alias Explorer.Chain.Cache.Helper, as: CacheHelper + alias Explorer.Chain.Celo.ElectionReward, as: CeloElectionReward require Logger @@ -327,8 +328,7 @@ defmodule Explorer.Chain.Address.Counters do AddressTransactionsGasUsageCounter.fetch(address) end - @spec address_limited_counters(Hash.t(), Keyword.t()) :: - {counter(), counter(), counter(), counter(), counter(), counter(), counter()} + @spec address_limited_counters(Hash.t(), Keyword.t()) :: %{atom() => counter} def address_limited_counters(address_hash, options) do cached_counters = Enum.reduce(@types, %{}, fn type, acc -> @@ -464,6 +464,19 @@ defmodule Explorer.Chain.Address.Counters do options ) + celo_election_rewards_count_task = + if Application.get_env(:explorer, :chain_type) == :celo do + configure_task( + :celo_election_rewards, + cached_counters, + CeloElectionReward.address_hash_to_rewards_query(address_hash), + address_hash, + options + ) + else + nil + end + map = [ validations_count_task, @@ -474,7 +487,8 @@ defmodule Explorer.Chain.Address.Counters do token_balances_count_task, logs_count_task, withdrawals_count_task, - internal_txs_count_task + internal_txs_count_task, + celo_election_rewards_count_task ] |> Enum.reject(&is_nil/1) |> Task.yield_many(:timer.seconds(1)) @@ -512,8 +526,7 @@ defmodule Explorer.Chain.Address.Counters do end) |> process_txs_counter() - {map[:validations], map[:txs], map[:token_transfers], map[:token_balances], map[:logs], map[:withdrawals], - map[:internal_txs]} + map end defp run_or_ignore({ok, _counter}, _type, _address_hash, _fun) when ok in [:up_to_date, :limit_value], do: nil diff --git a/apps/explorer/lib/explorer/chain/block.ex b/apps/explorer/lib/explorer/chain/block.ex index 785b5634da..60f60251cd 100644 --- a/apps/explorer/lib/explorer/chain/block.ex +++ b/apps/explorer/lib/explorer/chain/block.ex @@ -5,10 +5,19 @@ defmodule Explorer.Chain.Block.Schema do Changes in the schema should be reflected in the bulk import module: - Explorer.Chain.Import.Runner.Blocks """ + alias Explorer.Chain.{ + Address, + Block, + Hash, + PendingBlockOperation, + Transaction, + Wei, + Withdrawal + } - alias Explorer.Chain.{Address, Block, Hash, PendingBlockOperation, Transaction, Wei, Withdrawal} alias Explorer.Chain.Arbitrum.BatchBlock, as: ArbitrumBatchBlock alias Explorer.Chain.Block.{Reward, SecondDegreeRelation} + alias Explorer.Chain.Celo.EpochReward, as: CeloEpochReward alias Explorer.Chain.Optimism.TxnBatch, as: OptimismTxnBatch alias Explorer.Chain.ZkSync.BatchBlock, as: ZkSyncBatchBlock @@ -59,6 +68,19 @@ defmodule Explorer.Chain.Block.Schema do 2 ) + :celo -> + elem( + quote do + has_one(:celo_epoch_reward, CeloEpochReward, foreign_key: :block_hash, references: :hash) + + has_many(:celo_epoch_election_rewards, CeloEpochReward, + foreign_key: :block_hash, + references: :hash + ) + end, + 2 + ) + :arbitrum -> elem( quote do diff --git a/apps/explorer/lib/explorer/chain/cache/celo_core_contracts.ex b/apps/explorer/lib/explorer/chain/cache/celo_core_contracts.ex new file mode 100644 index 0000000000..42e86fc203 --- /dev/null +++ b/apps/explorer/lib/explorer/chain/cache/celo_core_contracts.ex @@ -0,0 +1,235 @@ +defmodule Explorer.Chain.Cache.CeloCoreContracts do + @moduledoc """ + Cache for Celo core contract addresses. + + This module operates with a `CELO_CORE_CONTRACTS` environment variable, which + contains the JSON map of core contract addresses on the Celo network. The + module provides functions to fetch the addresses of core contracts at a given + block. Additionally, it provides a function to obtain the state of a specific + contract by fetching the latest event for the contract at a given block. + + For details on the structure of the `CELO_CORE_CONTRACTS` environment + variable, see `app/explorer/lib/fetch_celo_core_contracts.ex`. + """ + @dialyzer :no_match + + require Logger + + alias EthereumJSONRPC + alias Explorer.Chain.Block + + @type contract_name :: String.t() + + @atom_to_contract_name %{ + accounts: "Accounts", + celo_token: "GoldToken", + election: "Election", + epoch_rewards: "EpochRewards", + locked_gold: "LockedGold", + reserve: "Reserve", + usd_token: "StableToken", + validators: "Validators", + governance: "Governance", + fee_handler: "FeeHandler", + gas_price_minimum: "GasPriceMinimum" + } + + @atom_to_contract_event_names %{ + fee_handler: %{ + fee_beneficiary_set: "FeeBeneficiarySet", + burn_fraction_set: "BurnFractionSet" + }, + epoch_rewards: %{ + carbon_offsetting_fund_set: "CarbonOffsettingFundSet" + } + } + + @doc """ + A map where keys are atoms representing contract types, and values are strings + representing the names of the contracts. + """ + @spec atom_to_contract_name() :: %{atom() => contract_name} + def atom_to_contract_name, do: @atom_to_contract_name + + @doc """ + A nested map where keys are atoms representing contract types, and values are + maps of event atoms to event names. + """ + @spec atom_to_contract_event_names() :: %{atom() => %{atom() => contract_name}} + def atom_to_contract_event_names, do: @atom_to_contract_event_names + + defp core_contracts, do: Application.get_env(:explorer, __MODULE__)[:contracts] + + @doc """ + Gets the specified event for a core contract at a given block number. + + ## Parameters + - `contract_atom`: The atom representing the contract. + - `event_atom`: The atom representing the event. + - `block_number`: The block number at which to fetch the event. + + ## Returns (one of the following) + - `{:ok, map() | nil}`: The event data if found, or `nil` if no event is found. + - `{:error, reason}`: An error tuple with the reason for the failure. + """ + @spec get_event(atom(), atom(), Block.block_number()) :: + {:ok, map() | nil} + | {:error, + :contract_atom_not_found + | :event_atom_not_found + | :contract_name_not_found + | :event_name_not_found + | :contract_address_not_found + | :event_does_not_exist} + def get_event(contract_atom, event_atom, block_number) do + with {:ok, address} <- get_address(contract_atom, block_number), + {:contract_atom, {:ok, contract_name}} <- + {:contract_atom, Map.fetch(@atom_to_contract_name, contract_atom)}, + {:event_atom, {:ok, event_name}} <- + { + :event_atom, + @atom_to_contract_event_names + |> Map.get(contract_atom, %{}) + |> Map.fetch(event_atom) + }, + {:events, {:ok, contract_name_to_addresses}} <- + {:events, Map.fetch(core_contracts(), "events")}, + {:contract_name, {:ok, contract_addresses}} <- + {:contract_name, Map.fetch(contract_name_to_addresses, contract_name)}, + {:contract_address, {:ok, contract_events}} <- + {:contract_address, Map.fetch(contract_addresses, address)}, + {:event_name, {:ok, event_updates}} <- + {:event_name, Map.fetch(contract_events, event_name)}, + current_event when not is_nil(current_event) <- + event_updates + |> Enum.take_while(&(&1["updated_at_block_number"] <= block_number)) + |> List.last() do + {:ok, current_event} + else + nil -> + {:ok, nil} + + {:contract_atom, :error} -> + Logger.error("Unknown contract atom: #{inspect(contract_atom)}") + {:error, :contract_atom_not_found} + + {:event_atom, :error} -> + Logger.error("Unknown event atom: #{inspect(event_atom)}") + {:error, :event_atom_not_found} + + {:events, :error} -> + raise "Missing `events` key in CELO core contracts JSON" + + {:contract_name, :error} -> + Logger.error(fn -> + [ + "Unknown name for contract atom: #{contract_atom}, ", + "ensure `CELO_CORE_CONTRACTS` env var is set ", + "and the provided JSON contains required key" + ] + end) + + {:error, :contract_name_not_found} + + {:event_name, :error} -> + Logger.error(fn -> + [ + "Unknown name for event atom: #{event_atom}, ", + "ensure `CELO_CORE_CONTRACTS` env var is set ", + "and the provided JSON contains required key" + ] + end) + + {:error, :event_name_not_found} + + nil -> + {:error, :event_does_not_exist} + + {:contract_address, :error} -> + Logger.error(fn -> + [ + "Unknown address for contract atom: #{contract_atom}, ", + "ensure `CELO_CORE_CONTRACTS` env var is set ", + "and the provided JSON contains required key" + ] + end) + + {:error, :contract_address_not_found} + + error -> + error + end + end + + defp get_address_updates(contract_atom) do + with {:atom, {:ok, contract_name}} <- + {:atom, Map.fetch(@atom_to_contract_name, contract_atom)}, + {:addresses, {:ok, contract_name_to_addresses}} <- + {:addresses, Map.fetch(core_contracts(), "addresses")}, + {:name, {:ok, address_updates}} <- + {:name, Map.fetch(contract_name_to_addresses, contract_name)} do + {:ok, address_updates} + else + {:atom, :error} -> + Logger.error("Unknown contract atom: #{inspect(contract_atom)}") + {:error, :contract_atom_not_found} + + {:addresses, :error} -> + raise "Missing `addresses` key in CELO core contracts JSON" + + {:name, :error} -> + Logger.error(fn -> + [ + "Unknown name for contract atom: #{contract_atom}, ", + "ensure `CELO_CORE_CONTRACTS` env var is set ", + "and the provided JSON contains required key" + ] + end) + + {:error, :contract_name_not_found} + end + end + + @doc """ + Gets the address of a core contract at a given block number. + + ## Parameters + - `contract_atom`: The atom representing the contract. + - `block_number`: The block number at which to fetch the address. + + ## Returns (one of the following) + - `{:ok, EthereumJSONRPC.address() | nil}`: The address of the contract, or `nil` if not found. + - `{:error, reason}`: An error tuple with the reason for the failure. + """ + @spec get_address(atom(), Block.block_number()) :: + {:ok, EthereumJSONRPC.address() | nil} + | {:error, + :contract_atom_not_found + | :contract_name_not_found + | :address_does_not_exist} + def get_address(contract_atom, block_number) do + with {:ok, address_updates} <- get_address_updates(contract_atom), + %{"address" => current_address} <- + address_updates + |> Enum.take_while(&(&1["updated_at_block_number"] <= block_number)) + |> List.last() do + {:ok, current_address} + else + nil -> + {:error, :address_does_not_exist} + + error -> + error + end + end + + def get_first_update_block_number(contract_atom) do + with {:ok, address_updates} <- get_address_updates(contract_atom), + %{"updated_at_block_number" => updated_at_block_number} <- + address_updates + |> Enum.sort_by(& &1["updated_at_block_number"]) + |> List.first() do + {:ok, updated_at_block_number} + end + end +end diff --git a/apps/explorer/lib/explorer/chain/celo/election_reward.ex b/apps/explorer/lib/explorer/chain/celo/election_reward.ex new file mode 100644 index 0000000000..f31898bf9d --- /dev/null +++ b/apps/explorer/lib/explorer/chain/celo/election_reward.ex @@ -0,0 +1,455 @@ +defmodule Explorer.Chain.Celo.ElectionReward do + @moduledoc """ + Represents the rewards distributed in an epoch election. Each reward has a + type, and each type of reward is paid in a specific token. The rewards are + paid to an account address and are also associated with another account + address. + + ## Reward Types and Addresses + + Here is the breakdown of what each address means for each type of reward: + + - `voter`: + - Account address: The voter address. + - Associated account address: The group address. + - `validator`: + - Account address: The validator address. + - Associated account address: The validator group address. + - `group`: + - Account address: The validator group address. + - Associated account address: The validator address that the reward was paid + on behalf of. + - `delegated_payment`: + - Account address: The beneficiary receiving the part of the reward on + behalf of the validator. + - Associated account address: The validator that set the delegation of a + part of their reward to some external address. + """ + + use Explorer.Schema + + import Explorer.PagingOptions, only: [default_paging_options: 0] + import Ecto.Query, only: [from: 2, where: 3] + import Explorer.Helper, only: [safe_parse_non_negative_integer: 1] + + alias Explorer.{Chain, PagingOptions} + alias Explorer.Chain.{Address, Block, Hash, Wei} + + @type type :: :voter | :validator | :group | :delegated_payment + @types_enum ~w(voter validator group delegated_payment)a + + @reward_type_string_to_atom %{ + "voter" => :voter, + "validator" => :validator, + "group" => :group, + "delegated-payment" => :delegated_payment + } + + @reward_type_atom_to_token_atom %{ + :voter => :celo_token, + :validator => :usd_token, + :group => :usd_token, + :delegated_payment => :usd_token + } + + @required_attrs ~w(amount type block_hash account_address_hash associated_account_address_hash)a + + @primary_key false + typed_schema "celo_election_rewards" do + field(:amount, Wei, null: false) + + field( + :type, + Ecto.Enum, + values: @types_enum, + null: false, + primary_key: true + ) + + belongs_to( + :block, + Block, + primary_key: true, + foreign_key: :block_hash, + references: :hash, + type: Hash.Full, + null: false + ) + + belongs_to( + :account_address, + Address, + primary_key: true, + foreign_key: :account_address_hash, + references: :hash, + type: Hash.Address, + null: false + ) + + belongs_to( + :associated_account_address, + Address, + primary_key: true, + foreign_key: :associated_account_address_hash, + references: :hash, + type: Hash.Address, + null: false + ) + + timestamps() + end + + @spec changeset( + Explorer.Chain.Celo.ElectionReward.t(), + map() + ) :: Ecto.Changeset.t() + def changeset(%__MODULE__{} = rewards, attrs) do + rewards + |> cast(attrs, @required_attrs) + |> validate_required(@required_attrs) + |> foreign_key_constraint(:block_hash) + |> foreign_key_constraint(:account_address_hash) + |> foreign_key_constraint(:associated_account_address_hash) + + # todo: do I need to set this unique constraint here? or it is redundant? + # |> unique_constraint( + # [:block_hash, :type, :account_address_hash, :associated_account_address_hash], + # name: :celo_election_rewards_pkey + # ) + end + + @doc """ + Returns the list of election reward types. + """ + @spec types() :: [type] + def types, do: @types_enum + + @doc """ + Converts a reward type string to its corresponding atom. + + ## Parameters + - `type_string` (`String.t()`): The string representation of the reward type. + + ## Returns + - `{:ok, type}` if the string is valid, `:error` otherwise. + + ## Examples + + iex> ElectionReward.type_from_string("voter") + {:ok, :voter} + + iex> ElectionReward.type_from_string("invalid") + :error + """ + @spec type_from_string(String.t()) :: {:ok, type} | :error + def type_from_string(type_string) do + Map.fetch(@reward_type_string_to_atom, type_string) + end + + @doc """ + Returns a map of reward type atoms to their corresponding token atoms. + + ## Returns + - A map where the keys are reward type atoms and the values are token atoms. + + ## Examples + + iex> ElectionReward.reward_type_atom_to_token_atom() + %{voter: :celo_token, validator: :usd_token, group: :usd_token, delegated_payment: :usd_token} + """ + @spec reward_type_atom_to_token_atom() :: %{type => atom()} + def reward_type_atom_to_token_atom, do: @reward_type_atom_to_token_atom + + @doc """ + Builds a query to aggregate rewards by type for a given block hash. + + ## Parameters + - `block_hash` (`Hash.Full.t()`): The block hash to filter rewards. + + ## Returns + - An Ecto query. + """ + @spec block_hash_to_aggregated_rewards_by_type_query(Hash.Full.t()) :: Ecto.Query.t() + def block_hash_to_aggregated_rewards_by_type_query(block_hash) do + from( + r in __MODULE__, + where: r.block_hash == ^block_hash, + select: {r.type, sum(r.amount), count(r)}, + group_by: r.type + ) + end + + @doc """ + Builds a query to get rewards by type for a given block hash. + + ## Parameters + - `block_hash` (`Hash.Full.t()`): The block hash to filter rewards. + - `reward_type` (`type`): The type of reward to filter. + + ## Returns + - An Ecto query. + """ + @spec block_hash_to_rewards_by_type_query(Hash.Full.t(), type) :: Ecto.Query.t() + def block_hash_to_rewards_by_type_query(block_hash, reward_type) do + from( + r in __MODULE__, + where: r.block_hash == ^block_hash and r.type == ^reward_type, + select: r, + order_by: [ + desc: :amount, + asc: :account_address_hash, + asc: :associated_account_address_hash + ] + ) + end + + @doc """ + Builds a query to get rewards by account address hash. + """ + @spec address_hash_to_rewards_query(Hash.Address.t()) :: Ecto.Query.t() + def address_hash_to_rewards_query(address_hash) do + from( + r in __MODULE__, + where: r.account_address_hash == ^address_hash, + select: r + ) + end + + @doc """ + Builds a query to get ordered rewards by account address hash. + + ## Parameters + - `address_hash` (`Hash.Address.t()`): The account address hash to filter + rewards. + + ## Returns + - An Ecto query. + """ + @spec address_hash_to_ordered_rewards_query(Hash.Address.t()) :: Ecto.Query.t() + def address_hash_to_ordered_rewards_query(address_hash) do + from( + r in __MODULE__, + join: b in assoc(r, :block), + as: :block, + preload: [block: b], + where: r.account_address_hash == ^address_hash, + select: r, + order_by: [ + desc: b.number, + desc: r.amount, + asc: r.associated_account_address_hash, + asc: r.type + ] + ) + end + + @doc """ + Makes Explorer.PagingOptions map for election rewards. + """ + @spec address_paging_options(map()) :: [Chain.paging_options()] + def block_paging_options(params) do + with %{ + "amount" => amount_string, + "account_address_hash" => account_address_hash_string, + "associated_account_address_hash" => associated_account_address_hash_string + } + when is_binary(amount_string) and + is_binary(account_address_hash_string) and + is_binary(associated_account_address_hash_string) <- params, + {amount, ""} <- Decimal.parse(amount_string), + {:ok, account_address_hash} <- Hash.Address.cast(account_address_hash_string), + {:ok, associated_account_address_hash} <- + Hash.Address.cast(associated_account_address_hash_string) do + [ + paging_options: %{ + default_paging_options() + | key: {amount, account_address_hash, associated_account_address_hash} + } + ] + else + _ -> + [paging_options: default_paging_options()] + end + end + + @doc """ + Makes Explorer.PagingOptions map for election rewards. + """ + @spec address_paging_options(map()) :: [Chain.paging_options()] + def address_paging_options(params) do + with %{ + "block_number" => block_number_string, + "amount" => amount_string, + "associated_account_address_hash" => associated_account_address_hash_string, + "type" => type_string + } + when is_binary(block_number_string) and + is_binary(amount_string) and + is_binary(associated_account_address_hash_string) and + is_binary(type_string) <- params, + {:ok, block_number} <- safe_parse_non_negative_integer(block_number_string), + {amount, ""} <- Decimal.parse(amount_string), + {:ok, associated_account_address_hash} <- + Hash.Address.cast(associated_account_address_hash_string), + {:ok, type} <- type_from_string(type_string) do + [ + paging_options: %{ + default_paging_options() + | key: {block_number, amount, associated_account_address_hash, type} + } + ] + else + _ -> + [paging_options: default_paging_options()] + end + end + + @doc """ + Paginates the given query based on the provided `PagingOptions`. + + ## Parameters + - `query` (`Ecto.Query.t()`): The query to paginate. + - `paging_options` (`PagingOptions.t()`): The pagination options. + + ## Returns + - An Ecto query with pagination applied. + """ + def paginate(query, %PagingOptions{key: nil}), do: query + + def paginate(query, %PagingOptions{key: {0 = _amount, account_address_hash, associated_account_address_hash}}) do + where( + query, + [reward], + reward.amount == 0 and + (reward.account_address_hash > ^account_address_hash or + (reward.account_address_hash == ^account_address_hash and + reward.associated_account_address_hash > ^associated_account_address_hash)) + ) + end + + def paginate(query, %PagingOptions{key: {amount, account_address_hash, associated_account_address_hash}}) do + where( + query, + [reward], + reward.amount < ^amount or + (reward.amount == ^amount and + reward.account_address_hash > ^account_address_hash) or + (reward.amount == ^amount and + reward.account_address_hash == ^account_address_hash and + reward.associated_account_address_hash > ^associated_account_address_hash) + ) + end + + def paginate(query, %PagingOptions{key: {0 = _block_number, 0 = _amount, associated_account_address_hash, type}}) do + where( + query, + [reward, block], + block.number == 0 and reward.amount == 0 and + (reward.associated_account_address_hash > ^associated_account_address_hash or + (reward.associated_account_address_hash == ^associated_account_address_hash and + reward.type > ^type)) + ) + end + + def paginate(query, %PagingOptions{key: {0 = _block_number, amount, associated_account_address_hash, type}}) do + where( + query, + [reward, block], + block.number == 0 and + (reward.amount < ^amount or + (reward.amount == ^amount and + reward.associated_account_address_hash > ^associated_account_address_hash) or + (reward.amount == ^amount and + reward.associated_account_address_hash == ^associated_account_address_hash and + reward.type > ^type)) + ) + end + + # credo:disable-for-next-line Credo.Check.Refactor.CyclomaticComplexity + def paginate(query, %PagingOptions{key: {block_number, 0 = _amount, associated_account_address_hash, type}}) do + where( + query, + [reward, block], + block.number < ^block_number or + (block.number == ^block_number and + reward.amount == 0 and + reward.associated_account_address_hash > ^associated_account_address_hash) or + (block.number == ^block_number and + reward.amount == 0 and + reward.associated_account_address_hash == ^associated_account_address_hash and + reward.type > ^type) + ) + end + + # credo:disable-for-next-line Credo.Check.Refactor.CyclomaticComplexity + def paginate(query, %PagingOptions{key: {block_number, amount, associated_account_address_hash, type}}) do + where( + query, + [reward, block], + block.number < ^block_number or + (block.number == ^block_number and + reward.amount < ^amount) or + (block.number == ^block_number and + reward.amount == ^amount and + reward.associated_account_address_hash > ^associated_account_address_hash) or + (block.number == ^block_number and + reward.amount == ^amount and + reward.associated_account_address_hash == ^associated_account_address_hash and + reward.type > ^type) + ) + end + + @doc """ + Converts an `ElectionReward` struct to paging parameters on the block view. + + ## Parameters + - `reward` (`%__MODULE__{}`): The election reward struct. + + ## Returns + - A map representing the block paging parameters. + + ## Examples + + iex> ElectionReward.to_block_paging_params(%ElectionReward{amount: 1000, account_address_hash: "0x123", associated_account_address_hash: "0x456"}) + %{"amount" => 1000, "account_address_hash" => "0x123", "associated_account_address_hash" => "0x456"} + """ + def to_block_paging_params(%__MODULE__{ + amount: amount, + account_address_hash: account_address_hash, + associated_account_address_hash: associated_account_address_hash + }) do + %{ + "amount" => amount, + "account_address_hash" => account_address_hash, + "associated_account_address_hash" => associated_account_address_hash + } + end + + @doc """ + Converts an `ElectionReward` struct to paging parameters on the address view. + + ## Parameters + - `reward` (`%__MODULE__{}`): The election reward struct. + + ## Returns + - A map representing the address paging parameters. + + ## Examples + + iex> ElectionReward.to_address_paging_params(%ElectionReward{block_number: 1, amount: 1000, associated_account_address_hash: "0x456", type: :voter}) + %{"block_number" => 1, "amount" => 1000, "associated_account_address_hash" => "0x456", "type" => :voter} + """ + def to_address_paging_params(%__MODULE__{ + block: %Block{number: block_number}, + amount: amount, + associated_account_address_hash: associated_account_address_hash, + type: type + }) do + %{ + "block_number" => block_number, + "amount" => amount, + "associated_account_address_hash" => associated_account_address_hash, + "type" => type + } + end +end diff --git a/apps/explorer/lib/explorer/chain/celo/epoch_reward.ex b/apps/explorer/lib/explorer/chain/celo/epoch_reward.ex new file mode 100644 index 0000000000..8356456f25 --- /dev/null +++ b/apps/explorer/lib/explorer/chain/celo/epoch_reward.ex @@ -0,0 +1,117 @@ +defmodule Explorer.Chain.Celo.EpochReward do + @moduledoc """ + Represents the distributions in the Celo epoch. Each log index points to a + token transfer event in the `TokenTransfer` relation. These include the + reserve bolster, community, and carbon offsetting transfers. + """ + use Explorer.Schema + + import Ecto.Query, only: [from: 2] + import Explorer.Chain, only: [select_repo: 1] + + alias Explorer.Chain.Celo.EpochReward + alias Explorer.Chain.{Block, Hash, TokenTransfer} + + @required_attrs ~w(block_hash)a + @optional_attrs ~w(reserve_bolster_transfer_log_index community_transfer_log_index carbon_offsetting_transfer_log_index)a + @allowed_attrs @required_attrs ++ @optional_attrs + + @primary_key false + typed_schema "celo_epoch_rewards" do + field(:reserve_bolster_transfer_log_index, :integer) + field(:community_transfer_log_index, :integer) + field(:carbon_offsetting_transfer_log_index, :integer) + field(:reserve_bolster_transfer, :any, virtual: true) :: TokenTransfer.t() | nil + field(:community_transfer, :any, virtual: true) :: TokenTransfer.t() | nil + field(:carbon_offsetting_transfer, :any, virtual: true) :: TokenTransfer.t() | nil + + belongs_to( + :block, + Block, + primary_key: true, + foreign_key: :block_hash, + references: :hash, + type: Hash.Full, + null: false + ) + + timestamps() + end + + @spec changeset(%__MODULE__{}, map()) :: Ecto.Changeset.t() + def changeset(%__MODULE__{} = rewards, attrs) do + rewards + |> cast(attrs, @allowed_attrs) + |> validate_required(@required_attrs) + |> foreign_key_constraint(:block_hash) + |> unique_constraint(:block_hash) + end + + @doc """ + Loads the token transfers for the given epoch reward. + + This function retrieves token transfers related to the specified epoch reward + by knowing the index of the log of the token transfer and populates the + virtual fields in the `EpochReward` struct. We manually preload token + transfers since Ecto does not support automatically preloading objects by + composite key (i.e., `log_index` and `block_hash`). + + ## Parameters + - `epoch_reward` (`EpochReward.t()`): The epoch reward struct. + - `options` (`Keyword.t()`): Optional parameters for selecting the repository. + + ## Returns + - `EpochReward.t()`: The epoch reward struct with the token transfers loaded. + + ## Example + + iex> epoch_reward = %Explorer.Chain.Celo.EpochReward{block_hash: "some_hash", reserve_bolster_transfer_log_index: 1} + iex> Explorer.Chain.Celo.EpochReward.load_token_transfers(epoch_reward) + %Explorer.Chain.Celo.EpochReward{ + block_hash: "some_hash", + reserve_bolster_transfer_log_index: 1, + reserve_bolster_transfer: %Explorer.Chain.TokenTransfer{log_index: 1, ...} + } + """ + + @spec load_token_transfers(EpochReward.t()) :: EpochReward.t() + def load_token_transfers( + %EpochReward{ + reserve_bolster_transfer_log_index: reserve_bolster_transfer_log_index, + community_transfer_log_index: community_transfer_log_index, + carbon_offsetting_transfer_log_index: carbon_offsetting_transfer_log_index + } = epoch_reward, + options \\ [] + ) do + virtual_field_to_log_index = [ + reserve_bolster_transfer: reserve_bolster_transfer_log_index, + community_transfer: community_transfer_log_index, + carbon_offsetting_transfer: carbon_offsetting_transfer_log_index + ] + + log_indexes = + virtual_field_to_log_index + |> Enum.map(&elem(&1, 1)) + |> Enum.reject(&is_nil/1) + + query = + from( + tt in TokenTransfer.only_consensus_transfers_query(), + where: tt.log_index in ^log_indexes and tt.block_hash == ^epoch_reward.block_hash, + select: {tt.log_index, tt}, + preload: [ + :token, + [from_address: [:names, :smart_contract, :proxy_implementations]], + [to_address: [:names, :smart_contract, :proxy_implementations]] + ] + ) + + log_index_to_token_transfer = query |> select_repo(options).all() |> Map.new() + + Enum.reduce(virtual_field_to_log_index, epoch_reward, fn + {field, log_index}, acc -> + token_transfer = Map.get(log_index_to_token_transfer, log_index) + Map.put(acc, field, token_transfer) + end) + end +end diff --git a/apps/explorer/lib/explorer/chain/celo/helper.ex b/apps/explorer/lib/explorer/chain/celo/helper.ex new file mode 100644 index 0000000000..0ede3c91fa --- /dev/null +++ b/apps/explorer/lib/explorer/chain/celo/helper.ex @@ -0,0 +1,94 @@ +defmodule Explorer.Chain.Celo.Helper do + @moduledoc """ + Common helper functions for Celo. + """ + + import Explorer.Chain.Cache.CeloCoreContracts, only: [atom_to_contract_name: 0] + + alias Explorer.Chain.Block + + @blocks_per_epoch 17_280 + @core_contract_atoms atom_to_contract_name() |> Map.keys() + + @doc """ + Returns the number of blocks per epoch in the Celo network. + """ + @spec blocks_per_epoch() :: non_neg_integer() + def blocks_per_epoch, do: @blocks_per_epoch + + defguard is_epoch_block_number(block_number) + when is_integer(block_number) and + block_number > 0 and + rem(block_number, @blocks_per_epoch) == 0 + + defguard is_core_contract_atom(atom) + when atom in @core_contract_atoms + + @doc """ + Validates if a block number is an epoch block number. + + ## Parameters + - `block_number` (`Block.block_number()`): The block number to validate. + + ## Returns + - `:ok` if the block number is an epoch block number. + - `{:error, :not_found}` if the block number is not an epoch block number. + + ## Examples + + iex> Explorer.Chain.Celo.Helper.validate_epoch_block_number(17280) + :ok + + iex> Explorer.Chain.Celo.Helper.validate_epoch_block_number(17281) + {:error, :not_found} + """ + @spec validate_epoch_block_number(Block.block_number()) :: :ok | {:error, :not_found} + def validate_epoch_block_number(block_number) when is_epoch_block_number(block_number), + do: :ok + + def validate_epoch_block_number(_block_number), do: {:error, :not_found} + + @doc """ + Checks if a block number belongs to a block that finalized an epoch. + + ## Parameters + - `block_number` (`Block.block_number()`): The block number to check. + + ## Returns + - `boolean()`: `true` if the block number is an epoch block number, `false` + otherwise. + + ## Examples + + iex> Explorer.Chain.Celo.Helper.epoch_block_number?(17280) + true + + iex> Explorer.Chain.Celo.Helper.epoch_block_number?(17281) + false + """ + @spec epoch_block_number?(block_number :: Block.block_number()) :: boolean() + def epoch_block_number?(block_number) when is_epoch_block_number(block_number), do: true + def epoch_block_number?(_), do: false + + @doc """ + Converts a block number to an epoch number. + + ## Parameters + - `block_number` (`Block.block_number()`): The block number to convert. + + ## Returns + - `non_neg_integer()`: The corresponding epoch number. + + ## Examples + + iex> Explorer.Chain.Celo.Helper.block_number_to_epoch_number(17280) + 1 + + iex> Explorer.Chain.Celo.Helper.block_number_to_epoch_number(17281) + 2 + """ + @spec block_number_to_epoch_number(block_number :: Block.block_number()) :: non_neg_integer() + def block_number_to_epoch_number(block_number) when is_integer(block_number) do + (block_number / @blocks_per_epoch) |> Float.ceil() |> trunc() + end +end diff --git a/apps/explorer/lib/explorer/chain/celo/pending_epoch_block_operation.ex b/apps/explorer/lib/explorer/chain/celo/pending_epoch_block_operation.ex new file mode 100644 index 0000000000..7a0152bcd1 --- /dev/null +++ b/apps/explorer/lib/explorer/chain/celo/pending_epoch_block_operation.ex @@ -0,0 +1,74 @@ +defmodule Explorer.Chain.Celo.PendingEpochBlockOperation do + @moduledoc """ + Tracks an epoch block that has pending operation. + """ + + use Explorer.Schema + + import Explorer.Chain, only: [add_fetcher_limit: 2] + alias Explorer.Chain.{Block, Hash} + alias Explorer.Repo + + @required_attrs ~w(block_hash)a + + @typedoc """ + * `block_hash` - the hash of the block that has pending epoch operations. + """ + @primary_key false + typed_schema "celo_pending_epoch_block_operations" do + belongs_to(:block, Block, + foreign_key: :block_hash, + primary_key: true, + references: :hash, + type: Hash.Full, + null: false + ) + + timestamps() + end + + def changeset(%__MODULE__{} = pending_ops, attrs) do + pending_ops + |> cast(attrs, @required_attrs) + |> validate_required(@required_attrs) + |> foreign_key_constraint(:block_hash) + |> unique_constraint(:block_hash, name: :pending_epoch_block_operations_pkey) + end + + @doc """ + Returns a stream of all blocks with unfetched epochs, using + the `celo_pending_epoch_block_operations` table. + + iex> unfetched = insert(:block, number: 1 * blocks_per_epoch()) + iex> insert(:celo_pending_epoch_block_operation, block: unfetched) + iex> {:ok, blocks} = PendingEpochBlockOperation.stream_epoch_blocks_with_unfetched_rewards( + ...> [], + ...> fn block, acc -> + ...> [block | acc] + ...> end + ...> ) + iex> [{unfetched.number, unfetched.hash}] == blocks + true + """ + @spec stream_epoch_blocks_with_unfetched_rewards( + initial :: accumulator, + reducer :: (entry :: term(), accumulator -> accumulator), + limited? :: boolean() + ) :: {:ok, accumulator} + when accumulator: term() + def stream_epoch_blocks_with_unfetched_rewards(initial, reducer, limited? \\ false) + when is_function(reducer, 2) do + query = + from( + op in __MODULE__, + join: block in assoc(op, :block), + select: %{block_number: block.number, block_hash: block.hash}, + where: block.consensus == true, + order_by: [desc: block.number] + ) + + query + |> add_fetcher_limit(limited?) + |> Repo.stream_reduce(initial, reducer) + end +end diff --git a/apps/explorer/lib/explorer/chain/celo/reader.ex b/apps/explorer/lib/explorer/chain/celo/reader.ex new file mode 100644 index 0000000000..d1c2dfff92 --- /dev/null +++ b/apps/explorer/lib/explorer/chain/celo/reader.ex @@ -0,0 +1,193 @@ +defmodule Explorer.Chain.Celo.Reader do + @moduledoc """ + Read functions for Celo modules. + """ + + import Ecto.Query, only: [limit: 2] + + import Explorer.Chain, + only: [ + select_repo: 1, + join_associations: 2, + default_paging_options: 0 + ] + + alias Explorer.Chain.Cache.CeloCoreContracts + alias Explorer.Chain.Celo.ElectionReward + alias Explorer.Chain.{Hash, Token, Wei} + + @election_reward_types ElectionReward.types() + @default_paging_options default_paging_options() + + @doc """ + Retrieves election rewards associated with a given address hash. + + ## Parameters + - `address_hash` (`Hash.Address.t()`): The address hash to search for election + rewards. + - `options` (`Keyword.t()`): Optional parameters for fetching data. + + ## Returns + - `[ElectionReward.t()]`: A list of election rewards associated with the + address hash. + + ## Examples + + iex> address_hash = %Hash.Address{ + ...> byte_count: 20, + ...> bytes: <<0x1d1f7f0e1441c37e28b89e0b5e1edbbd34d77649 :: size(160)>> + ...> } + iex> Explorer.Chain.Celo.Reader.address_hash_to_election_rewards(address_hash) + [%ElectionReward{}, ...] + """ + @spec address_hash_to_election_rewards( + Hash.Address.t(), + Keyword.t() + ) :: [ElectionReward.t()] + def address_hash_to_election_rewards(address_hash, options \\ []) do + necessity_by_association = Keyword.get(options, :necessity_by_association, %{}) + paging_options = Keyword.get(options, :paging_options, @default_paging_options) + + address_hash + |> ElectionReward.address_hash_to_ordered_rewards_query() + |> ElectionReward.paginate(paging_options) + |> limit(^paging_options.page_size) + |> join_associations(necessity_by_association) + |> select_repo(options).all() + end + + @doc """ + Retrieves election rewards by block hash and reward type. + + ## Parameters + - `block_hash` (`Hash.t()`): The block hash to search for election rewards. + - `reward_type` (`ElectionReward.type()`): The type of reward to filter. + - `options` (`Keyword.t()`): Optional parameters for fetching data. + + ## Returns + - `[ElectionReward.t()]`: A list of election rewards filtered by block hash + and reward type. + + ## Examples + + iex> block_hash = %Hash.Full{ + ...> byte_count: 32, + ...> bytes: <<0x9fc76417374aa880d4449a1f7f31ec597f00b1f6f3dd2d66f4c9c6c445836d8b :: big-integer-size(32)-unit(8)>> + ...> } + iex> Explorer.Chain.Celo.Reader.block_hash_to_election_rewards_by_type(block_hash, :voter_reward) + [%ElectionReward{}, ...] + """ + @spec block_hash_to_election_rewards_by_type( + Hash.t(), + ElectionReward.type(), + Keyword.t() + ) :: [ElectionReward.t()] + def block_hash_to_election_rewards_by_type(block_hash, reward_type, options \\ []) + when reward_type in @election_reward_types do + necessity_by_association = Keyword.get(options, :necessity_by_association, %{}) + paging_options = Keyword.get(options, :paging_options, @default_paging_options) + + block_hash + |> ElectionReward.block_hash_to_rewards_by_type_query(reward_type) + |> ElectionReward.paginate(paging_options) + |> limit(^paging_options.page_size) + |> join_associations(necessity_by_association) + |> select_repo(options).all() + end + + @doc """ + Retrieves aggregated election rewards by block hash. + + ## Parameters + - `block_hash` (`Hash.Full.t()`): The block hash to aggregate election + rewards. + - `options` (`Keyword.t()`): Optional parameters for fetching data. + + ## Returns + - `%{atom() => Wei.t() | nil}`: A map of aggregated election rewards by type. + + ## Examples + + iex> block_hash = %Hash.Full{ + ...> byte_count: 32, + ...> bytes: <<0x9fc76417374aa880d4449a1f7f31ec597f00b1f6f3dd2d66f4c9c6c445836d8b :: big-integer-size(32)-unit(8)>> + ...> } + iex> Explorer.Chain.Celo.Reader.block_hash_to_aggregated_election_rewards_by_type(block_hash) + %{voter_reward: %{total: %Decimal{}, count: 2}, ...} + """ + @spec block_hash_to_aggregated_election_rewards_by_type( + Hash.Full.t(), + Keyword.t() + ) :: %{atom() => Wei.t() | nil} + def block_hash_to_aggregated_election_rewards_by_type(block_hash, options \\ []) do + reward_type_to_token = + block_hash_to_election_reward_token_addresses_by_type( + block_hash, + options + ) + + reward_type_to_aggregated_rewards = + block_hash + |> ElectionReward.block_hash_to_aggregated_rewards_by_type_query() + |> select_repo(options).all() + |> Map.new(fn {type, total, count} -> + {type, %{total: total, count: count}} + end) + + ElectionReward.types() + |> Map.new(&{&1, %{total: Decimal.new(0), count: 0}}) + |> Map.merge(reward_type_to_aggregated_rewards) + |> Map.new(fn {type, aggregated_reward} -> + token = Map.get(reward_type_to_token, type) + aggregated_reward_with_token = Map.put(aggregated_reward, :token, token) + {type, aggregated_reward_with_token} + end) + end + + # Retrieves the token for each type of election reward on the given block. + # + # ## Parameters + # - `block_hash` (`Hash.Full.t()`): The block hash to search for token + # addresses. + # - `options` (`Keyword.t()`): Optional parameters for fetching data. + # + # ## Returns + # - `%{atom() => Token.t() | nil}`: A map of reward types to token. + # + # ## Examples + # + # iex> block_hash = %Hash.Full{ + # ...> byte_count: 32, + # ...> bytes: <<0x9fc76417374aa880d4449a1f7f31ec597f00b1f6f3dd2d66f4c9c6c445836d8b :: big-integer-size(32)-unit(8)>> + # ...> } + # iex> Explorer.Chain.Celo.Reader.block_hash_to_election_reward_token_addresses_by_type(block_hash) + # %{voter_reward: %Token{}, ...} + @spec block_hash_to_election_reward_token_addresses_by_type( + Hash.Full.t(), + Keyword.t() + ) :: %{atom() => Token.t() | nil} + defp block_hash_to_election_reward_token_addresses_by_type(block_hash, options) do + contract_address_hash_to_atom = + ElectionReward.reward_type_atom_to_token_atom() + |> Map.values() + |> Map.new(fn token_atom -> + {:ok, contract_address_hash} = CeloCoreContracts.get_address(token_atom, block_hash) + {contract_address_hash, token_atom} + end) + + token_atom_to_token = + contract_address_hash_to_atom + |> Map.keys() + |> Token.get_by_contract_address_hashes(options) + |> Map.new(fn token -> + hash = to_string(token.contract_address_hash) + atom = contract_address_hash_to_atom[hash] + {atom, token} + end) + + ElectionReward.reward_type_atom_to_token_atom() + |> Map.new(fn {reward_type_atom, token_atom} -> + {reward_type_atom, Map.get(token_atom_to_token, token_atom)} + end) + end +end diff --git a/apps/explorer/lib/explorer/chain/celo/validator_group_vote.ex b/apps/explorer/lib/explorer/chain/celo/validator_group_vote.ex new file mode 100644 index 0000000000..dcb639e49f --- /dev/null +++ b/apps/explorer/lib/explorer/chain/celo/validator_group_vote.ex @@ -0,0 +1,76 @@ +defmodule Explorer.Chain.Celo.ValidatorGroupVote do + @moduledoc """ + Represents the information about a vote for a validator group made by an + account. + """ + + use Explorer.Schema + + alias Explorer.Chain.{Address, Block, Hash, Transaction} + + @types_enum ~w(activated revoked)a + @required_attrs ~w(account_address_hash group_address_hash type transaction_hash block_hash block_number)a + + @typedoc """ + * `account_address_hash` - the address of the account that made the vote. + * `group_address_hash` - the address of the validator group that + was voted for. + * `type` - whether this vote is `activated` or `revoked`. + * `block_number` - the block number of the vote. + * `block_hash` - the hash of the block that contains the vote. + * `transaction_hash` - the hash of the transaction that made the vote. + """ + @primary_key false + typed_schema "celo_validator_group_votes" do + belongs_to(:account_address, Address, + foreign_key: :account_address_hash, + references: :hash, + type: Hash.Address + ) + + belongs_to(:group_address, Address, + foreign_key: :group_address_hash, + references: :hash, + type: Hash.Address + ) + + field(:type, Ecto.Enum, + values: @types_enum, + null: false + ) + + field(:block_number, :integer, null: false) + + belongs_to(:block, Block, + foreign_key: :block_hash, + primary_key: true, + references: :hash, + type: Hash.Full, + null: false + ) + + belongs_to(:transaction, Transaction, + foreign_key: :transaction_hash, + primary_key: true, + references: :hash, + type: Hash.Full, + null: false + ) + + timestamps() + end + + @spec changeset( + __MODULE__.t(), + :invalid | %{optional(:__struct__) => none, optional(atom | binary) => any} + ) :: Ecto.Changeset.t() + def changeset(%__MODULE__{} = vote, attrs \\ %{}) do + vote + |> cast(attrs, @required_attrs) + |> validate_required(@required_attrs) + |> foreign_key_constraint(:account_address_hash) + |> foreign_key_constraint(:group_address_hash) + |> foreign_key_constraint(:block_hash) + |> foreign_key_constraint(:transaction_hash) + end +end diff --git a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex index 8ef328d55b..0af2896b2c 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex @@ -24,6 +24,9 @@ defmodule Explorer.Chain.Import.Runner.Blocks do Transaction } + alias Explorer.Chain.Celo.Helper, as: CeloHelper + alias Explorer.Chain.Celo.PendingEpochBlockOperation, as: CeloPendingEpochBlockOperation + alias Explorer.Chain.Block.Reward alias Explorer.Chain.Import.Runner alias Explorer.Chain.Import.Runner.Address.CurrentTokenBalances @@ -207,11 +210,32 @@ defmodule Explorer.Chain.Import.Runner.Blocks do :blocks_update_token_holder_counts ) end) + |> chain_type_dependent_import( + :celo, + &Multi.run(&1, :celo_pending_epoch_block_operations, fn repo, %{blocks: blocks} -> + Instrumenter.block_import_stage_runner( + fn -> + celo_pending_epoch_block_operations(repo, blocks, insert_options) + end, + :address_referencing, + :blocks, + :celo_pending_epoch_block_operations + ) + end) + ) end @impl Runner def timeout, do: @timeout + def chain_type_dependent_import(multi, chain_type, multi_run) do + if Application.get_env(:explorer, :chain_type) == chain_type do + multi_run.(multi) + else + multi + end + end + defp fork_transactions(%{ repo: repo, timeout: timeout, @@ -916,4 +940,24 @@ defmodule Explorer.Chain.Import.Runner.Blocks do blocks end end + + defp celo_pending_epoch_block_operations(repo, inserted_blocks, %{timeout: timeout, timestamps: timestamps}) do + ordered_epoch_blocks = + inserted_blocks + |> Enum.filter(fn block -> CeloHelper.epoch_block_number?(block.number) && block.consensus end) + |> Enum.map(&%{block_hash: &1.hash}) + |> Enum.sort_by(& &1.block_hash) + |> Enum.dedup_by(& &1.block_hash) + + Import.insert_changes_list( + repo, + ordered_epoch_blocks, + conflict_target: :block_hash, + on_conflict: :nothing, + for: CeloPendingEpochBlockOperation, + returning: true, + timeout: timeout, + timestamps: timestamps + ) + end end diff --git a/apps/explorer/lib/explorer/chain/import/runner/celo/election_rewards.ex b/apps/explorer/lib/explorer/chain/import/runner/celo/election_rewards.ex new file mode 100644 index 0000000000..b7ff06f781 --- /dev/null +++ b/apps/explorer/lib/explorer/chain/import/runner/celo/election_rewards.ex @@ -0,0 +1,93 @@ +defmodule Explorer.Chain.Import.Runner.Celo.ElectionRewards do + @moduledoc """ + Bulk imports `t:Explorer.Chain.Celo.ElectionReward.t/0`. + """ + + require Ecto.Query + + alias Ecto.{Changeset, Multi, Repo} + alias Explorer.Chain.Celo.ElectionReward + alias Explorer.Chain.Import + alias Explorer.Prometheus.Instrumenter + + @behaviour Import.Runner + + # milliseconds + @timeout 60_000 + + @type imported :: [ElectionReward.t()] + + @impl Import.Runner + def ecto_schema_module, do: ElectionReward + + @impl Import.Runner + def option_key, do: :celo_election_rewards + + @impl Import.Runner + @spec imported_table_row() :: %{:value_description => binary(), :value_type => binary()} + def imported_table_row do + %{ + value_type: "[#{ecto_schema_module()}.t()]", + value_description: "List of `t:#{ecto_schema_module()}.t/0`s" + } + end + + @impl Import.Runner + @spec run(Multi.t(), list(), map()) :: Multi.t() + def run(multi, changes_list, %{timestamps: timestamps} = options) do + insert_options = + options + |> Map.get(option_key(), %{}) + |> Map.take(~w(on_conflict timeout)a) + |> Map.put_new(:timeout, @timeout) + |> Map.put(:timestamps, timestamps) + + Multi.run(multi, :insert_celo_election_rewards, fn repo, _ -> + Instrumenter.block_import_stage_runner( + fn -> insert(repo, changes_list, insert_options) end, + :block_referencing, + :celo_election_rewards, + :celo_election_rewards + ) + end) + end + + @impl Import.Runner + def timeout, do: @timeout + + @spec insert(Repo.t(), [map()], %{required(:timeout) => timeout(), required(:timestamps) => Import.timestamps()}) :: + {:ok, [ElectionReward.t()]} + | {:error, [Changeset.t()]} + def insert(repo, changes_list, %{timeout: timeout, timestamps: timestamps} = _options) when is_list(changes_list) do + # Enforce Celo.Epoch.ElectionReward ShareLocks order (see docs: sharelock.md) + ordered_changes_list = + Enum.sort_by( + changes_list, + &{ + &1.block_hash, + &1.type, + &1.account_address_hash, + &1.associated_account_address_hash + } + ) + + {:ok, inserted} = + Import.insert_changes_list( + repo, + ordered_changes_list, + for: ElectionReward, + returning: true, + timeout: timeout, + timestamps: timestamps, + conflict_target: [ + :block_hash, + :type, + :account_address_hash, + :associated_account_address_hash + ], + on_conflict: :replace_all + ) + + {:ok, inserted} + end +end diff --git a/apps/explorer/lib/explorer/chain/import/runner/celo/epoch_rewards.ex b/apps/explorer/lib/explorer/chain/import/runner/celo/epoch_rewards.ex new file mode 100644 index 0000000000..7c7cd5752e --- /dev/null +++ b/apps/explorer/lib/explorer/chain/import/runner/celo/epoch_rewards.ex @@ -0,0 +1,139 @@ +defmodule Explorer.Chain.Import.Runner.Celo.EpochRewards do + @moduledoc """ + Bulk imports `t:Explorer.Chain.Celo.EpochReward.t/0`. + """ + + require Ecto.Query + + alias Ecto.{Changeset, Multi, Repo} + alias Explorer.Chain.Celo.{EpochReward, PendingEpochBlockOperation} + alias Explorer.Chain.Import + alias Explorer.Prometheus.Instrumenter + + import Ecto.Query + + @behaviour Import.Runner + + # milliseconds + @timeout 60_000 + + @type imported :: [EpochReward.t()] + + @impl Import.Runner + def ecto_schema_module, do: EpochReward + + @impl Import.Runner + def option_key, do: :celo_epoch_rewards + + @impl Import.Runner + @spec imported_table_row() :: %{:value_description => binary(), :value_type => binary()} + def imported_table_row do + %{ + value_type: "[#{ecto_schema_module()}.t()]", + value_description: "List of `t:#{ecto_schema_module()}.t/0`s" + } + end + + @impl Import.Runner + @spec run(Multi.t(), list(), map()) :: Multi.t() + def run(multi, changes_list, %{timestamps: timestamps} = options) do + insert_options = + options + |> Map.get(option_key(), %{}) + |> Map.take(~w(on_conflict timeout)a) + |> Map.put_new(:timeout, @timeout) + |> Map.put(:timestamps, timestamps) + + multi + |> Multi.run(:acquire_pending_epoch_block_operations, fn repo, _ -> + Instrumenter.block_import_stage_runner( + fn -> acquire_pending_epoch_block_operations(repo, changes_list) end, + :block_pending, + :celo_epoch_rewards, + :acquire_pending_epoch_block_operations + ) + end) + |> Multi.run(:insert_celo_epoch_rewards, fn repo, _ -> + Instrumenter.block_import_stage_runner( + fn -> insert(repo, changes_list, insert_options) end, + :block_referencing, + :celo_epoch_rewards, + :celo_epoch_rewards + ) + end) + |> Multi.run( + :delete_pending_epoch_block_operations, + fn repo, + %{ + acquire_pending_epoch_block_operations: pending_block_hashes + } -> + Instrumenter.block_import_stage_runner( + fn -> delete_pending_epoch_block_operations(repo, pending_block_hashes) end, + :block_pending, + :celo_epoch_rewards, + :delete_pending_epoch_block_operations + ) + end + ) + end + + @impl Import.Runner + def timeout, do: @timeout + + @spec insert(Repo.t(), [map()], %{required(:timeout) => timeout(), required(:timestamps) => Import.timestamps()}) :: + {:ok, [EpochReward.t()]} + | {:error, [Changeset.t()]} + def insert(repo, changes_list, %{timeout: timeout, timestamps: timestamps} = _options) when is_list(changes_list) do + # Enforce Celo.EpochReward ShareLocks order (see docs: sharelock.md) + ordered_changes_list = Enum.sort_by(changes_list, & &1.block_hash) + + {:ok, inserted} = + Import.insert_changes_list( + repo, + ordered_changes_list, + for: EpochReward, + returning: true, + timeout: timeout, + timestamps: timestamps, + conflict_target: :block_hash, + on_conflict: :replace_all + ) + + {:ok, inserted} + end + + def acquire_pending_epoch_block_operations(repo, changes_list) do + block_hashes = Enum.map(changes_list, & &1.block_hash) + + query = + from( + pending_ops in PendingEpochBlockOperation, + where: pending_ops.block_hash in ^block_hashes, + select: pending_ops.block_hash, + # Enforce PendingBlockOperation ShareLocks order (see docs: sharelocks.md) + order_by: [asc: pending_ops.block_hash], + lock: "FOR UPDATE" + ) + + {:ok, repo.all(query)} + end + + def delete_pending_epoch_block_operations(repo, block_hashes) do + delete_query = + from( + pending_ops in PendingEpochBlockOperation, + where: pending_ops.block_hash in ^block_hashes + ) + + try do + # ShareLocks order already enforced by + # `acquire_pending_epoch_block_operations` (see docs: sharelocks.md) + {_count, deleted} = repo.delete_all(delete_query, []) + + {:ok, deleted} + rescue + postgrex_error in Postgrex.Error -> + {:error, %{exception: postgrex_error, pending_hashes: block_hashes}} + end + end +end diff --git a/apps/explorer/lib/explorer/chain/import/runner/celo/validator_group_votes.ex b/apps/explorer/lib/explorer/chain/import/runner/celo/validator_group_votes.ex new file mode 100644 index 0000000000..618b0bae52 --- /dev/null +++ b/apps/explorer/lib/explorer/chain/import/runner/celo/validator_group_votes.ex @@ -0,0 +1,87 @@ +defmodule Explorer.Chain.Import.Runner.Celo.ValidatorGroupVotes do + @moduledoc """ + Bulk imports `t:Explorer.Chain.Celo.ValidatorGroupVote.t/0`. + """ + + require Ecto.Query + + alias Ecto.{Changeset, Multi, Repo} + alias Explorer.Chain.Celo.ValidatorGroupVote + alias Explorer.Chain.Import + alias Explorer.Prometheus.Instrumenter + + @behaviour Import.Runner + + # milliseconds + @timeout 60_000 + + @type imported :: [ValidatorGroupVote.t()] + + @impl Import.Runner + def ecto_schema_module, do: ValidatorGroupVote + + @impl Import.Runner + def option_key, do: :celo_validator_group_votes + + @impl Import.Runner + @spec imported_table_row() :: %{:value_description => binary(), :value_type => binary()} + def imported_table_row do + %{ + value_type: "[#{ecto_schema_module()}.t()]", + value_description: "List of `t:#{ecto_schema_module()}.t/0`s" + } + end + + @impl Import.Runner + @spec run(Multi.t(), list(), map()) :: Multi.t() + def run(multi, changes_list, %{timestamps: timestamps} = options) do + insert_options = + options + |> Map.get(option_key(), %{}) + |> Map.take(~w(on_conflict timeout)a) + |> Map.put_new(:timeout, @timeout) + |> Map.put(:timestamps, timestamps) + + Multi.run(multi, :insert_celo_validator_group_votes, fn repo, _ -> + Instrumenter.block_import_stage_runner( + fn -> insert(repo, changes_list, insert_options) end, + :block_referencing, + :celo_validator_group_votes, + :celo_validator_group_votes + ) + end) + end + + @impl Import.Runner + def timeout, do: @timeout + + @spec insert(Repo.t(), [map()], %{required(:timeout) => timeout(), required(:timestamps) => Import.timestamps()}) :: + {:ok, [ValidatorGroupVote.t()]} + | {:error, [Changeset.t()]} + def insert(repo, changes_list, %{timeout: timeout, timestamps: timestamps} = _options) when is_list(changes_list) do + # Enforce Celo.Epoch.ValidatorGroupVote ShareLocks order (see docs: sharelock.md) + ordered_changes_list = + Enum.sort_by( + changes_list, + &{ + &1.transaction_hash, + &1.account_address_hash, + &1.group_address_hash + } + ) + + {:ok, inserted} = + Import.insert_changes_list( + repo, + ordered_changes_list, + for: ValidatorGroupVote, + returning: true, + timeout: timeout, + timestamps: timestamps, + conflict_target: :transaction_hash, + on_conflict: :replace_all + ) + + {:ok, inserted} + end +end diff --git a/apps/explorer/lib/explorer/chain/import/runner/logs.ex b/apps/explorer/lib/explorer/chain/import/runner/logs.ex index 7c0e591a0f..cb8466f2b2 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/logs.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/logs.ex @@ -65,13 +65,23 @@ defmodule Explorer.Chain.Import.Runner.Logs do on_conflict = Map.get_lazy(options, :on_conflict, &default_on_conflict/0) # Enforce Log ShareLocks order (see docs: sharelocks.md) - ordered_changes_list = Enum.sort_by(changes_list, &{&1.transaction_hash, &1.block_hash, &1.index}) + ordered_changes_list = + case Application.get_env(:explorer, :chain_type) do + :celo -> Enum.sort_by(changes_list, &{&1.block_hash, &1.index}) + _ -> Enum.sort_by(changes_list, &{&1.transaction_hash, &1.block_hash, &1.index}) + end + + conflict_target = + case Application.get_env(:explorer, :chain_type) do + :celo -> [:index, :block_hash] + _ -> [:transaction_hash, :index, :block_hash] + end {:ok, _} = Import.insert_changes_list( repo, ordered_changes_list, - conflict_target: [:transaction_hash, :index, :block_hash], + conflict_target: conflict_target, on_conflict: on_conflict, for: Log, returning: true, @@ -81,32 +91,65 @@ defmodule Explorer.Chain.Import.Runner.Logs do end defp default_on_conflict do - from( - log in Log, - update: [ - set: [ - address_hash: fragment("EXCLUDED.address_hash"), - data: fragment("EXCLUDED.data"), - first_topic: fragment("EXCLUDED.first_topic"), - second_topic: fragment("EXCLUDED.second_topic"), - third_topic: fragment("EXCLUDED.third_topic"), - fourth_topic: fragment("EXCLUDED.fourth_topic"), - # Don't update `index` as it is part of the composite primary key and used for the conflict target - # Don't update `transaction_hash` as it is part of the composite primary key and used for the conflict target - inserted_at: fragment("LEAST(?, EXCLUDED.inserted_at)", log.inserted_at), - updated_at: fragment("GREATEST(?, EXCLUDED.updated_at)", log.updated_at) - ] - ], - where: - fragment( - "(EXCLUDED.address_hash, EXCLUDED.data, EXCLUDED.first_topic, EXCLUDED.second_topic, EXCLUDED.third_topic, EXCLUDED.fourth_topic) IS DISTINCT FROM (?, ?, ?, ?, ?, ?)", - log.address_hash, - log.data, - log.first_topic, - log.second_topic, - log.third_topic, - log.fourth_topic + case Application.get_env(:explorer, :chain_type) do + :celo -> + from( + log in Log, + update: [ + set: [ + address_hash: fragment("EXCLUDED.address_hash"), + data: fragment("EXCLUDED.data"), + first_topic: fragment("EXCLUDED.first_topic"), + second_topic: fragment("EXCLUDED.second_topic"), + third_topic: fragment("EXCLUDED.third_topic"), + fourth_topic: fragment("EXCLUDED.fourth_topic"), + # Don't update `index` as it is part of the composite primary key and used for the conflict target + transaction_hash: fragment("EXCLUDED.transaction_hash"), + inserted_at: fragment("LEAST(?, EXCLUDED.inserted_at)", log.inserted_at), + updated_at: fragment("GREATEST(?, EXCLUDED.updated_at)", log.updated_at) + ] + ], + where: + fragment( + "(EXCLUDED.address_hash, EXCLUDED.data, EXCLUDED.first_topic, EXCLUDED.second_topic, EXCLUDED.third_topic, EXCLUDED.fourth_topic, EXCLUDED.transaction_hash) IS DISTINCT FROM (?, ?, ?, ?, ?, ?, ?)", + log.address_hash, + log.data, + log.first_topic, + log.second_topic, + log.third_topic, + log.fourth_topic, + log.transaction_hash + ) ) - ) + + _ -> + from( + log in Log, + update: [ + set: [ + address_hash: fragment("EXCLUDED.address_hash"), + data: fragment("EXCLUDED.data"), + first_topic: fragment("EXCLUDED.first_topic"), + second_topic: fragment("EXCLUDED.second_topic"), + third_topic: fragment("EXCLUDED.third_topic"), + fourth_topic: fragment("EXCLUDED.fourth_topic"), + # Don't update `index` as it is part of the composite primary key and used for the conflict target + # Don't update `transaction_hash` as it is part of the composite primary key and used for the conflict target + inserted_at: fragment("LEAST(?, EXCLUDED.inserted_at)", log.inserted_at), + updated_at: fragment("GREATEST(?, EXCLUDED.updated_at)", log.updated_at) + ] + ], + where: + fragment( + "(EXCLUDED.address_hash, EXCLUDED.data, EXCLUDED.first_topic, EXCLUDED.second_topic, EXCLUDED.third_topic, EXCLUDED.fourth_topic) IS DISTINCT FROM (?, ?, ?, ?, ?, ?)", + log.address_hash, + log.data, + log.first_topic, + log.second_topic, + log.third_topic, + log.fourth_topic + ) + ) + end end end diff --git a/apps/explorer/lib/explorer/chain/import/runner/token_transfers.ex b/apps/explorer/lib/explorer/chain/import/runner/token_transfers.ex index 2dac07465e..acbbf31592 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/token_transfers.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/token_transfers.ex @@ -61,13 +61,23 @@ defmodule Explorer.Chain.Import.Runner.TokenTransfers do on_conflict = Map.get_lazy(options, :on_conflict, &default_on_conflict/0) # Enforce TokenTransfer ShareLocks order (see docs: sharelocks.md) - ordered_changes_list = Enum.sort_by(changes_list, &{&1.transaction_hash, &1.block_hash, &1.log_index}) + ordered_changes_list = + case Application.get_env(:explorer, :chain_type) do + :celo -> Enum.sort_by(changes_list, &{&1.block_hash, &1.log_index}) + _ -> Enum.sort_by(changes_list, &{&1.transaction_hash, &1.block_hash, &1.log_index}) + end + + conflict_target = + case Application.get_env(:explorer, :chain_type) do + :celo -> [:log_index, :block_hash] + _ -> [:transaction_hash, :log_index, :block_hash] + end {:ok, inserted} = Import.insert_changes_list( repo, ordered_changes_list, - conflict_target: [:transaction_hash, :log_index, :block_hash], + conflict_target: conflict_target, on_conflict: on_conflict, for: TokenTransfer, returning: true, @@ -79,34 +89,70 @@ defmodule Explorer.Chain.Import.Runner.TokenTransfers do end defp default_on_conflict do - from( - token_transfer in TokenTransfer, - update: [ - set: [ - # Don't update `transaction_hash` as it is part of the composite primary key and used for the conflict target - # Don't update `log_index` as it is part of the composite primary key and used for the conflict target - amount: fragment("EXCLUDED.amount"), - from_address_hash: fragment("EXCLUDED.from_address_hash"), - to_address_hash: fragment("EXCLUDED.to_address_hash"), - token_contract_address_hash: fragment("EXCLUDED.token_contract_address_hash"), - token_ids: fragment("EXCLUDED.token_ids"), - token_type: fragment("EXCLUDED.token_type"), - block_consensus: fragment("EXCLUDED.block_consensus"), - inserted_at: fragment("LEAST(?, EXCLUDED.inserted_at)", token_transfer.inserted_at), - updated_at: fragment("GREATEST(?, EXCLUDED.updated_at)", token_transfer.updated_at) - ] - ], - where: - fragment( - "(EXCLUDED.amount, EXCLUDED.from_address_hash, EXCLUDED.to_address_hash, EXCLUDED.token_contract_address_hash, EXCLUDED.token_ids, EXCLUDED.token_type, EXCLUDED.block_consensus) IS DISTINCT FROM (?, ?, ?, ?, ?, ?, ?)", - token_transfer.amount, - token_transfer.from_address_hash, - token_transfer.to_address_hash, - token_transfer.token_contract_address_hash, - token_transfer.token_ids, - token_transfer.token_type, - token_transfer.block_consensus + case Application.get_env(:explorer, :chain_type) do + :celo -> + from( + token_transfer in TokenTransfer, + update: [ + set: [ + # Don't update `log_index` as it is part of the composite primary + # key and used for the conflict target + transaction_hash: fragment("EXCLUDED.transaction_hash"), + amount: fragment("EXCLUDED.amount"), + from_address_hash: fragment("EXCLUDED.from_address_hash"), + to_address_hash: fragment("EXCLUDED.to_address_hash"), + token_contract_address_hash: fragment("EXCLUDED.token_contract_address_hash"), + token_ids: fragment("EXCLUDED.token_ids"), + token_type: fragment("EXCLUDED.token_type"), + block_consensus: fragment("EXCLUDED.block_consensus"), + inserted_at: fragment("LEAST(?, EXCLUDED.inserted_at)", token_transfer.inserted_at), + updated_at: fragment("GREATEST(?, EXCLUDED.updated_at)", token_transfer.updated_at) + ] + ], + where: + fragment( + "(EXCLUDED.amount, EXCLUDED.from_address_hash, EXCLUDED.to_address_hash, EXCLUDED.token_contract_address_hash, EXCLUDED.token_ids, EXCLUDED.token_type, EXCLUDED.block_consensus, EXCLUDED.transaction_hash) IS DISTINCT FROM (?, ?, ?, ?, ?, ?, ?, ?)", + token_transfer.amount, + token_transfer.from_address_hash, + token_transfer.to_address_hash, + token_transfer.token_contract_address_hash, + token_transfer.token_ids, + token_transfer.token_type, + token_transfer.block_consensus, + token_transfer.transaction_hash + ) ) - ) + + _ -> + from( + token_transfer in TokenTransfer, + update: [ + set: [ + # Don't update `transaction_hash` as it is part of the composite primary key and used for the conflict target + # Don't update `log_index` as it is part of the composite primary key and used for the conflict target + amount: fragment("EXCLUDED.amount"), + from_address_hash: fragment("EXCLUDED.from_address_hash"), + to_address_hash: fragment("EXCLUDED.to_address_hash"), + token_contract_address_hash: fragment("EXCLUDED.token_contract_address_hash"), + token_ids: fragment("EXCLUDED.token_ids"), + token_type: fragment("EXCLUDED.token_type"), + block_consensus: fragment("EXCLUDED.block_consensus"), + inserted_at: fragment("LEAST(?, EXCLUDED.inserted_at)", token_transfer.inserted_at), + updated_at: fragment("GREATEST(?, EXCLUDED.updated_at)", token_transfer.updated_at) + ] + ], + where: + fragment( + "(EXCLUDED.amount, EXCLUDED.from_address_hash, EXCLUDED.to_address_hash, EXCLUDED.token_contract_address_hash, EXCLUDED.token_ids, EXCLUDED.token_type, EXCLUDED.block_consensus) IS DISTINCT FROM (?, ?, ?, ?, ?, ?, ?)", + token_transfer.amount, + token_transfer.from_address_hash, + token_transfer.to_address_hash, + token_transfer.token_contract_address_hash, + token_transfer.token_ids, + token_transfer.token_type, + token_transfer.block_consensus + ) + ) + end end end diff --git a/apps/explorer/lib/explorer/chain/import/runner/transactions.ex b/apps/explorer/lib/explorer/chain/import/runner/transactions.ex index 1b1772afd9..aee467977f 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/transactions.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/transactions.ex @@ -107,6 +107,7 @@ defmodule Explorer.Chain.Import.Runner.Transactions do ) end + # todo: avoid code duplication case Application.compile_env(:explorer, :chain_type) do :suave -> defp default_on_conflict do @@ -360,6 +361,83 @@ defmodule Explorer.Chain.Import.Runner.Transactions do ) end + :celo -> + defp default_on_conflict do + from( + transaction in Transaction, + update: [ + set: [ + block_hash: fragment("EXCLUDED.block_hash"), + old_block_hash: transaction.block_hash, + block_number: fragment("EXCLUDED.block_number"), + block_consensus: fragment("EXCLUDED.block_consensus"), + block_timestamp: fragment("EXCLUDED.block_timestamp"), + created_contract_address_hash: fragment("EXCLUDED.created_contract_address_hash"), + created_contract_code_indexed_at: fragment("EXCLUDED.created_contract_code_indexed_at"), + cumulative_gas_used: fragment("EXCLUDED.cumulative_gas_used"), + error: fragment("EXCLUDED.error"), + from_address_hash: fragment("EXCLUDED.from_address_hash"), + gas: fragment("EXCLUDED.gas"), + gas_price: fragment("EXCLUDED.gas_price"), + gas_used: fragment("EXCLUDED.gas_used"), + index: fragment("EXCLUDED.index"), + input: fragment("EXCLUDED.input"), + nonce: fragment("EXCLUDED.nonce"), + r: fragment("EXCLUDED.r"), + s: fragment("EXCLUDED.s"), + status: fragment("EXCLUDED.status"), + to_address_hash: fragment("EXCLUDED.to_address_hash"), + v: fragment("EXCLUDED.v"), + value: fragment("EXCLUDED.value"), + earliest_processing_start: fragment("EXCLUDED.earliest_processing_start"), + revert_reason: fragment("EXCLUDED.revert_reason"), + max_priority_fee_per_gas: fragment("EXCLUDED.max_priority_fee_per_gas"), + max_fee_per_gas: fragment("EXCLUDED.max_fee_per_gas"), + type: fragment("EXCLUDED.type"), + # Don't update `hash` as it is part of the primary key and used for the conflict target + inserted_at: fragment("LEAST(?, EXCLUDED.inserted_at)", transaction.inserted_at), + updated_at: fragment("GREATEST(?, EXCLUDED.updated_at)", transaction.updated_at), + # Celo custom fields + gas_token_contract_address_hash: fragment("EXCLUDED.gas_token_contract_address_hash"), + gas_fee_recipient_address_hash: fragment("EXCLUDED.gas_fee_recipient_address_hash"), + gateway_fee: fragment("EXCLUDED.gateway_fee") + ] + ], + where: + fragment( + "(EXCLUDED.block_hash, EXCLUDED.block_number, EXCLUDED.block_consensus, EXCLUDED.block_timestamp, EXCLUDED.created_contract_address_hash, EXCLUDED.created_contract_code_indexed_at, EXCLUDED.cumulative_gas_used, EXCLUDED.from_address_hash, EXCLUDED.gas, EXCLUDED.gas_price, EXCLUDED.gas_used, EXCLUDED.index, EXCLUDED.input, EXCLUDED.nonce, EXCLUDED.r, EXCLUDED.s, EXCLUDED.status, EXCLUDED.to_address_hash, EXCLUDED.v, EXCLUDED.value, EXCLUDED.earliest_processing_start, EXCLUDED.revert_reason, EXCLUDED.max_priority_fee_per_gas, EXCLUDED.max_fee_per_gas, EXCLUDED.type, EXCLUDED.gas_token_contract_address_hash, EXCLUDED.gas_fee_recipient_address_hash, EXCLUDED.gateway_fee) IS DISTINCT FROM (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + transaction.block_hash, + transaction.block_number, + transaction.block_consensus, + transaction.block_timestamp, + transaction.created_contract_address_hash, + transaction.created_contract_code_indexed_at, + transaction.cumulative_gas_used, + transaction.from_address_hash, + transaction.gas, + transaction.gas_price, + transaction.gas_used, + transaction.index, + transaction.input, + transaction.nonce, + transaction.r, + transaction.s, + transaction.status, + transaction.to_address_hash, + transaction.v, + transaction.value, + transaction.earliest_processing_start, + transaction.revert_reason, + transaction.max_priority_fee_per_gas, + transaction.max_fee_per_gas, + transaction.type, + transaction.gas_token_contract_address_hash, + transaction.gas_fee_recipient_address_hash, + transaction.gateway_fee + ) + ) + end + _ -> defp default_on_conflict do from( diff --git a/apps/explorer/lib/explorer/chain/import/stage/block_referencing.ex b/apps/explorer/lib/explorer/chain/import/stage/block_referencing.ex index 0d830810ed..0d339a78d3 100644 --- a/apps/explorer/lib/explorer/chain/import/stage/block_referencing.ex +++ b/apps/explorer/lib/explorer/chain/import/stage/block_referencing.ex @@ -7,6 +7,7 @@ defmodule Explorer.Chain.Import.Stage.BlockReferencing do alias Explorer.Chain.Import.{Runner, Stage} @behaviour Stage + @default_runners [ Runner.Transaction.Forks, Runner.Logs, @@ -17,92 +18,74 @@ defmodule Explorer.Chain.Import.Stage.BlockReferencing do Runner.Withdrawals ] - @optimism_runners [ - Runner.Optimism.FrameSequences, - Runner.Optimism.FrameSequenceBlobs, - Runner.Optimism.TxnBatches, - Runner.Optimism.OutputRoots, - Runner.Optimism.DisputeGames, - Runner.Optimism.Deposits, - Runner.Optimism.Withdrawals, - Runner.Optimism.WithdrawalEvents - ] - - @polygon_edge_runners [ - Runner.PolygonEdge.Deposits, - Runner.PolygonEdge.DepositExecutes, - Runner.PolygonEdge.Withdrawals, - Runner.PolygonEdge.WithdrawalExits - ] - - @polygon_zkevm_runners [ - Runner.PolygonZkevm.LifecycleTransactions, - Runner.PolygonZkevm.TransactionBatches, - Runner.PolygonZkevm.BatchTransactions, - Runner.PolygonZkevm.BridgeL1Tokens, - Runner.PolygonZkevm.BridgeOperations - ] - - @zksync_runners [ - Runner.ZkSync.LifecycleTransactions, - Runner.ZkSync.TransactionBatches, - Runner.ZkSync.BatchTransactions, - Runner.ZkSync.BatchBlocks - ] - - @shibarium_runners [ - Runner.Shibarium.BridgeOperations - ] - - @ethereum_runners [ - Runner.Beacon.BlobTransactions - ] - - @arbitrum_runners [ - Runner.Arbitrum.Messages, - Runner.Arbitrum.LifecycleTransactions, - Runner.Arbitrum.L1Executions, - Runner.Arbitrum.L1Batches, - Runner.Arbitrum.BatchBlocks, - Runner.Arbitrum.BatchTransactions, - Runner.Arbitrum.DaMultiPurposeRecords - ] + @extra_runners_by_chain_type %{ + optimism: [ + Runner.Optimism.FrameSequences, + Runner.Optimism.FrameSequenceBlobs, + Runner.Optimism.TxnBatches, + Runner.Optimism.OutputRoots, + Runner.Optimism.DisputeGames, + Runner.Optimism.Deposits, + Runner.Optimism.Withdrawals, + Runner.Optimism.WithdrawalEvents + ], + polygon_edge: [ + Runner.PolygonEdge.Deposits, + Runner.PolygonEdge.DepositExecutes, + Runner.PolygonEdge.Withdrawals, + Runner.PolygonEdge.WithdrawalExits + ], + polygon_zkevm: [ + Runner.PolygonZkevm.LifecycleTransactions, + Runner.PolygonZkevm.TransactionBatches, + Runner.PolygonZkevm.BatchTransactions, + Runner.PolygonZkevm.BridgeL1Tokens, + Runner.PolygonZkevm.BridgeOperations + ], + zksync: [ + Runner.ZkSync.LifecycleTransactions, + Runner.ZkSync.TransactionBatches, + Runner.ZkSync.BatchTransactions, + Runner.ZkSync.BatchBlocks + ], + shibarium: [ + Runner.Shibarium.BridgeOperations + ], + ethereum: [ + Runner.Beacon.BlobTransactions + ], + arbitrum: [ + Runner.Arbitrum.Messages, + Runner.Arbitrum.LifecycleTransactions, + Runner.Arbitrum.L1Executions, + Runner.Arbitrum.L1Batches, + Runner.Arbitrum.BatchBlocks, + Runner.Arbitrum.BatchTransactions, + Runner.Arbitrum.DaMultiPurposeRecords + ], + celo: [ + Runner.Celo.ValidatorGroupVotes, + Runner.Celo.ElectionRewards, + Runner.Celo.EpochRewards + ] + } @impl Stage def runners do - case Application.get_env(:explorer, :chain_type) do - :optimism -> - @default_runners ++ @optimism_runners - - :polygon_edge -> - @default_runners ++ @polygon_edge_runners - - :polygon_zkevm -> - @default_runners ++ @polygon_zkevm_runners + chain_type = Application.get_env(:explorer, :chain_type) + chain_type_runners = Map.get(@extra_runners_by_chain_type, chain_type, []) - :shibarium -> - @default_runners ++ @shibarium_runners - - :ethereum -> - @default_runners ++ @ethereum_runners - - :zksync -> - @default_runners ++ @zksync_runners - - :arbitrum -> - @default_runners ++ @arbitrum_runners - - _ -> - @default_runners - end + @default_runners ++ chain_type_runners end @impl Stage def all_runners do - @default_runners ++ - @ethereum_runners ++ - @optimism_runners ++ - @polygon_edge_runners ++ @polygon_zkevm_runners ++ @shibarium_runners ++ @zksync_runners ++ @arbitrum_runners + all_extra_runners = + @extra_runners_by_chain_type + |> Map.values() + |> Enum.concat() + + @default_runners ++ all_extra_runners end @impl Stage diff --git a/apps/explorer/lib/explorer/chain/log.ex b/apps/explorer/lib/explorer/chain/log.ex index 7147f20986..7ef3588844 100644 --- a/apps/explorer/lib/explorer/chain/log.ex +++ b/apps/explorer/lib/explorer/chain/log.ex @@ -1,18 +1,111 @@ +defmodule Explorer.Chain.Log.Schema do + @moduledoc false + + alias Explorer.Chain.{ + Address, + Block, + Data, + Hash, + Transaction + } + + # In certain situations, like on Polygon, multiple logs may share the same + # index within a single block due to a RPC node bug. To prevent system crashes + # due to not unique primary keys, we've included `transaction_hash` in the + # primary key. + # + # However, on Celo, logs may exist where `transaction_hash` equals block_hash. + # In these instances, we set `transaction_hash` to `nil`. This action, though, + # violates the primary key constraint. To resolve this issue, we've excluded + # `transaction_hash` from the composite primary key when dealing with `:celo` + # chain type. + @transaction_field (case Application.compile_env(:explorer, :chain_type) do + :celo -> + quote do + [ + belongs_to(:transaction, Transaction, + foreign_key: :transaction_hash, + references: :hash, + type: Hash.Full + ) + ] + end + + _ -> + quote do + [ + belongs_to(:transaction, Transaction, + foreign_key: :transaction_hash, + primary_key: true, + references: :hash, + type: Hash.Full, + null: false + ) + ] + end + end) + + defmacro generate do + quote do + @primary_key false + typed_schema "logs" do + field(:data, Data, null: false) + field(:first_topic, Hash.Full) + field(:second_topic, Hash.Full) + field(:third_topic, Hash.Full) + field(:fourth_topic, Hash.Full) + field(:index, :integer, primary_key: true, null: false) + field(:block_number, :integer) + + timestamps() + + belongs_to(:address, Address, foreign_key: :address_hash, references: :hash, type: Hash.Address, null: false) + + belongs_to(:block, Block, + foreign_key: :block_hash, + primary_key: true, + references: :hash, + type: Hash.Full, + null: false + ) + + unquote_splicing(@transaction_field) + end + end + end +end + defmodule Explorer.Chain.Log do @moduledoc "Captures a Web3 log entry generated by a transaction" use Explorer.Schema + require Explorer.Chain.Log.Schema require Logger alias ABI.{Event, FunctionSelector} alias Explorer.{Chain, Repo} - alias Explorer.Chain.{Address, Block, ContractMethod, Data, Hash, Log, TokenTransfer, Transaction} + alias Explorer.Chain.{ContractMethod, Hash, Log, TokenTransfer, Transaction} alias Explorer.Chain.SmartContract.Proxy alias Explorer.SmartContract.SigProviderInterface - @required_attrs ~w(address_hash data block_hash index transaction_hash)a + @required_attrs ~w(address_hash data block_hash index)a + |> (&(case Application.compile_env(:explorer, :chain_type) do + :celo -> + &1 + + _ -> + [:transaction_hash | &1] + end)).() + @optional_attrs ~w(first_topic second_topic third_topic fourth_topic block_number)a + |> (&(case Application.compile_env(:explorer, :chain_type) do + :celo -> + [:transaction_hash | &1] + + _ -> + &1 + end)).() @typedoc """ * `address` - address of contract that generate the event @@ -28,36 +121,7 @@ defmodule Explorer.Chain.Log do * `transaction_hash` - foreign key for `transaction`. * `index` - index of the log entry within the block """ - @primary_key false - typed_schema "logs" do - field(:data, Data, null: false) - field(:first_topic, Hash.Full) - field(:second_topic, Hash.Full) - field(:third_topic, Hash.Full) - field(:fourth_topic, Hash.Full) - field(:index, :integer, primary_key: true, null: false) - field(:block_number, :integer) - - timestamps() - - belongs_to(:address, Address, foreign_key: :address_hash, references: :hash, type: Hash.Address, null: false) - - belongs_to(:transaction, Transaction, - foreign_key: :transaction_hash, - primary_key: true, - references: :hash, - type: Hash.Full, - null: false - ) - - belongs_to(:block, Block, - foreign_key: :block_hash, - primary_key: true, - references: :hash, - type: Hash.Full, - null: false - ) - end + Explorer.Chain.Log.Schema.generate() @doc """ `address_hash` and `transaction_hash` are converted to `t:Explorer.Chain.Hash.t/0`. diff --git a/apps/explorer/lib/explorer/chain/smart_contract.ex b/apps/explorer/lib/explorer/chain/smart_contract.ex index 71a2a0064f..121b3c6863 100644 --- a/apps/explorer/lib/explorer/chain/smart_contract.ex +++ b/apps/explorer/lib/explorer/chain/smart_contract.ex @@ -37,15 +37,24 @@ defmodule Explorer.Chain.SmartContract do @typep api? :: {:api?, true | false} @burn_address_hash_string "0x0000000000000000000000000000000000000000" + @dead_address_hash_string "0x000000000000000000000000000000000000dEaD" @doc """ Returns burn address hash """ - @spec burn_address_hash_string() :: String.t() + @spec burn_address_hash_string() :: EthereumJSONRPC.address() def burn_address_hash_string do @burn_address_hash_string end + @doc """ + Returns dead address hash + """ + @spec dead_address_hash_string() :: EthereumJSONRPC.address() + def dead_address_hash_string do + @dead_address_hash_string + end + @default_sorting [desc: :id] @typedoc """ diff --git a/apps/explorer/lib/explorer/chain/smart_contract/proxy.ex b/apps/explorer/lib/explorer/chain/smart_contract/proxy.ex index 4edd81c6be..35c874c6ee 100644 --- a/apps/explorer/lib/explorer/chain/smart_contract/proxy.ex +++ b/apps/explorer/lib/explorer/chain/smart_contract/proxy.ex @@ -42,7 +42,7 @@ defmodule Explorer.Chain.SmartContract.Proxy do @get_implementation_signature "aaf10f42" # bb82aa5e = keccak256(comptrollerImplementation()) Compound protocol proxy pattern @comptroller_implementation_signature "bb82aa5e" - # aaf10f42 = keccak256(getAddress(bytes32)) + # 21f8a721 = keccak256(getAddress(bytes32)) @get_address_signature "21f8a721" @typep options :: [{:api?, true | false}, {:proxy_without_abi?, true | false}] diff --git a/apps/explorer/lib/explorer/chain/token_transfer.ex b/apps/explorer/lib/explorer/chain/token_transfer.ex index 2201b62305..bafd52e39c 100644 --- a/apps/explorer/lib/explorer/chain/token_transfer.ex +++ b/apps/explorer/lib/explorer/chain/token_transfer.ex @@ -1,3 +1,112 @@ +defmodule Explorer.Chain.TokenTransfer.Schema do + @moduledoc """ + Models token transfers. + + Changes in the schema should be reflected in the bulk import module: + - Explorer.Chain.Import.Runner.TokenTransfers + """ + + alias Explorer.Chain.{ + Address, + Block, + Hash, + Transaction + } + + alias Explorer.Chain.Token.Instance + + # Remove `transaction_hash` from primary key for `:celo` chain type. See + # `Explorer.Chain.Log.Schema` for more details. + @transaction_field (case Application.compile_env(:explorer, :chain_type) do + :celo -> + quote do + [ + belongs_to(:transaction, Transaction, + foreign_key: :transaction_hash, + references: :hash, + type: Hash.Full + ) + ] + end + + _ -> + quote do + [ + belongs_to(:transaction, Transaction, + foreign_key: :transaction_hash, + primary_key: true, + references: :hash, + type: Hash.Full, + null: false + ) + ] + end + end) + + defmacro generate do + quote do + @primary_key false + typed_schema "token_transfers" do + field(:amount, :decimal) + field(:block_number, :integer) :: Block.block_number() + field(:log_index, :integer, primary_key: true, null: false) + field(:amounts, {:array, :decimal}) + field(:token_ids, {:array, :decimal}) + field(:token_id, :decimal, virtual: true) + field(:index_in_batch, :integer, virtual: true) + field(:reverse_index_in_batch, :integer, virtual: true) + field(:token_decimals, :decimal, virtual: true) + field(:token_type, :string) + field(:block_consensus, :boolean) + + belongs_to(:from_address, Address, + foreign_key: :from_address_hash, + references: :hash, + type: Hash.Address, + null: false + ) + + belongs_to(:to_address, Address, + foreign_key: :to_address_hash, + references: :hash, + type: Hash.Address, + null: false + ) + + belongs_to( + :token_contract_address, + Address, + foreign_key: :token_contract_address_hash, + references: :hash, + type: Hash.Address, + null: false + ) + + belongs_to(:block, Block, + foreign_key: :block_hash, + primary_key: true, + references: :hash, + type: Hash.Full, + null: false + ) + + has_many( + :instances, + Instance, + foreign_key: :token_contract_address_hash, + references: :token_contract_address_hash + ) + + has_one(:token, through: [:token_contract_address, :token]) + + timestamps() + + unquote_splicing(@transaction_field) + end + end + end +end + defmodule Explorer.Chain.TokenTransfer do @moduledoc """ Represents a token transfer between addresses for a given token. @@ -24,11 +133,12 @@ defmodule Explorer.Chain.TokenTransfer do use Explorer.Schema + require Explorer.Chain.TokenTransfer.Schema + import Ecto.Changeset alias Explorer.Chain - alias Explorer.Chain.{Address, Block, DenormalizationHelper, Hash, Log, TokenTransfer, Transaction} - alias Explorer.Chain.Token.Instance + alias Explorer.Chain.{DenormalizationHelper, Hash, Log, TokenTransfer} alias Explorer.{PagingOptions, Repo} @default_paging_options %PagingOptions{page_size: 50} @@ -66,68 +176,24 @@ defmodule Explorer.Chain.TokenTransfer do * `:reverse_index_in_batch` - Reverse index of the token transfer in the ERC-1155 batch transfer, last element index is 1 * `:block_consensus` - Consensus of the block that the transfer took place """ - @primary_key false - typed_schema "token_transfers" do - field(:amount, :decimal) - field(:block_number, :integer) :: Block.block_number() - field(:log_index, :integer, primary_key: true, null: false) - field(:amounts, {:array, :decimal}) - field(:token_ids, {:array, :decimal}) - field(:token_id, :decimal, virtual: true) - field(:index_in_batch, :integer, virtual: true) - field(:reverse_index_in_batch, :integer, virtual: true) - field(:token_decimals, :decimal, virtual: true) - field(:token_type, :string) - field(:block_consensus, :boolean) - - belongs_to(:from_address, Address, - foreign_key: :from_address_hash, - references: :hash, - type: Hash.Address, - null: false - ) - - belongs_to(:to_address, Address, foreign_key: :to_address_hash, references: :hash, type: Hash.Address, null: false) - - belongs_to( - :token_contract_address, - Address, - foreign_key: :token_contract_address_hash, - references: :hash, - type: Hash.Address, - null: false - ) + Explorer.Chain.TokenTransfer.Schema.generate() - belongs_to(:transaction, Transaction, - foreign_key: :transaction_hash, - primary_key: true, - references: :hash, - type: Hash.Full, - null: false - ) - - belongs_to(:block, Block, - foreign_key: :block_hash, - primary_key: true, - references: :hash, - type: Hash.Full, - null: false - ) + @required_attrs ~w(block_number log_index from_address_hash to_address_hash token_contract_address_hash block_hash token_type)a + |> (&(case Application.compile_env(:explorer, :chain_type) do + :celo -> + &1 - has_many( - :instances, - Instance, - foreign_key: :token_contract_address_hash, - references: :token_contract_address_hash - ) - - has_one(:token, through: [:token_contract_address, :token]) - - timestamps() - end - - @required_attrs ~w(block_number log_index from_address_hash to_address_hash token_contract_address_hash transaction_hash block_hash token_type)a + _ -> + [:transaction_hash | &1] + end)).() @optional_attrs ~w(amount amounts token_ids block_consensus)a + |> (&(case Application.compile_env(:explorer, :chain_type) do + :celo -> + [:transaction_hash | &1] + + _ -> + &1 + end)).() @doc false def changeset(%TokenTransfer{} = struct, params \\ %{}) do diff --git a/apps/explorer/lib/explorer/chain/transaction.ex b/apps/explorer/lib/explorer/chain/transaction.ex index 3b24bc2354..2954241ab7 100644 --- a/apps/explorer/lib/explorer/chain/transaction.ex +++ b/apps/explorer/lib/explorer/chain/transaction.ex @@ -122,6 +122,28 @@ defmodule Explorer.Chain.Transaction.Schema do 2 ) + :celo -> + elem( + quote do + field(:gateway_fee, Wei) + + belongs_to(:gas_fee_recipient, Address, + foreign_key: :gas_fee_recipient_address_hash, + references: :hash, + type: Hash.Address + ) + + belongs_to(:gas_token_contract_address, Address, + foreign_key: :gas_token_contract_address_hash, + references: :hash, + type: Hash.Address + ) + + has_one(:gas_token, through: [:gas_token_contract_address, :token]) + end, + 2 + ) + :arbitrum -> elem( quote do @@ -297,6 +319,9 @@ defmodule Explorer.Chain.Transaction do :arbitrum -> ~w(gas_used_for_l1)a + :celo -> + ~w(gateway_fee gas_fee_recipient_address_hash gas_token_contract_address_hash)a + _ -> ~w()a end) diff --git a/apps/explorer/lib/explorer/helper.ex b/apps/explorer/lib/explorer/helper.ex index 5f33572cfa..e4d2bc6e46 100644 --- a/apps/explorer/lib/explorer/helper.ex +++ b/apps/explorer/lib/explorer/helper.ex @@ -33,6 +33,36 @@ defmodule Explorer.Helper do |> TypeDecoder.decode_raw(types) end + @doc """ + Takes an Ethereum hash and converts it to a standard 20-byte address by + truncating the leading zeroes. If the input is `nil`, it returns the burn + address. + + ## Parameters + - `address_hash` (`EthereumJSONRPC.hash()` | `nil`): The full address hash to + be truncated, or `nil`. + + ## Returns + - `EthereumJSONRPC.address()`: The truncated address or the burn address if + the input is `nil`. + + ## Examples + + iex> truncate_address_hash("0x000000000000000000000000abcdef1234567890abcdef1234567890abcdef") + "0xabcdef1234567890abcdef1234567890abcdef" + + iex> truncate_address_hash(nil) + "0x0000000000000000000000000000000000000000" + """ + @spec truncate_address_hash(EthereumJSONRPC.hash() | nil) :: EthereumJSONRPC.address() + def truncate_address_hash(address_hash) + + def truncate_address_hash(nil), do: burn_address_hash_string() + + def truncate_address_hash("0x000000000000000000000000" <> truncated_hash) do + "0x#{truncated_hash}" + end + def parse_integer(integer_string) when is_binary(integer_string) do case Integer.parse(integer_string) do {integer, ""} -> integer @@ -182,10 +212,4 @@ defmodule Explorer.Helper do true -> :eq end end - - def truncate_address_hash(nil), do: burn_address_hash_string() - - def truncate_address_hash("0x000000000000000000000000" <> truncated_hash) do - "0x#{truncated_hash}" - end end diff --git a/apps/explorer/lib/explorer/paging_options.ex b/apps/explorer/lib/explorer/paging_options.ex index 303bd53c37..081d160c3d 100644 --- a/apps/explorer/lib/explorer/paging_options.ex +++ b/apps/explorer/lib/explorer/paging_options.ex @@ -31,4 +31,14 @@ defmodule Explorer.PagingOptions do asc_order: false, batch_key: nil ] + + @page_size 50 + + def page_size do + @page_size + end + + def default_paging_options do + %__MODULE__{page_size: @page_size + 1} + end end diff --git a/apps/explorer/lib/explorer/repo.ex b/apps/explorer/lib/explorer/repo.ex index 8105224b4f..a0ae857fa4 100644 --- a/apps/explorer/lib/explorer/repo.ex +++ b/apps/explorer/lib/explorer/repo.ex @@ -177,6 +177,16 @@ defmodule Explorer.Repo do end end + defmodule Celo do + use Ecto.Repo, + otp_app: :explorer, + adapter: Ecto.Adapters.Postgres + + def init(_, opts) do + ConfigHelper.init_repo_module(__MODULE__, opts) + end + end + defmodule RSK do use Ecto.Repo, otp_app: :explorer, diff --git a/apps/explorer/lib/fetch_celo_core_contracts.ex b/apps/explorer/lib/fetch_celo_core_contracts.ex new file mode 100644 index 0000000000..a1a12884ab --- /dev/null +++ b/apps/explorer/lib/fetch_celo_core_contracts.ex @@ -0,0 +1,236 @@ +defmodule Mix.Tasks.FetchCeloCoreContracts do + @moduledoc """ + Fetch the addresses of celo core contracts: `mix help celo-contracts` + """ + @shortdoc "Fetches the addresses of celo core contracts" + + use Mix.Task + + import Explorer.Helper, + only: [ + decode_data: 2, + truncate_address_hash: 1 + ] + + alias Mix.Task + + alias EthereumJSONRPC.Logs + alias Explorer.Chain.Cache.CeloCoreContracts + alias Indexer.Helper, as: IndexerHelper + + @registry_proxy_contract_address "0x000000000000000000000000000000000000ce10" + @registry_updated_event_signature "0x4166d073a7a5e704ce0db7113320f88da2457f872d46dc020c805c562c1582a0" + @carbon_offsetting_fund_set_event_signature "0xe296227209b47bb8f4a76768ebd564dcde1c44be325a5d262f27c1fd4fd4538b" + @fee_beneficiary_set_event_signature "0xf7015098f8d6fa48f0560725ffd5f81bf687ee5ac45153b590bdcb04648bbdd3" + @burn_fraction_set_event_signature "0x41c679f4bcdc2c95f79a3647e2237162d9763d86685ef6c667781230c8ba9157" + + @chunk_size 200_000 + @max_request_retries 3 + + def run(_) do + Task.run("app.start") + json_rpc_named_arguments = Application.get_env(:explorer, :json_rpc_named_arguments) + atom_to_contract_name = CeloCoreContracts.atom_to_contract_name() + atom_to_contract_event_names = CeloCoreContracts.atom_to_contract_event_names() + contract_names = atom_to_contract_name |> Map.values() + {:ok, latest_block_number} = EthereumJSONRPC.fetch_block_number_by_tag("latest", json_rpc_named_arguments) + + core_contract_addresses = + 0..latest_block_number + |> fetch_logs_by_chunks( + fn chunk_start, chunk_end -> + [ + Logs.request( + 0, + %{ + from_block: chunk_start, + to_block: chunk_end, + address: @registry_proxy_contract_address, + topics: [@registry_updated_event_signature] + } + ) + ] + end, + json_rpc_named_arguments + ) + |> Enum.reduce(%{}, fn log, acc -> + [contract_name] = decode_data(log.data, [:string]) + new_contract_address = truncate_address_hash(log.third_topic) + + entry = %{ + address: new_contract_address, + updated_at_block_number: log.block_number + } + + if contract_name in contract_names do + acc + |> Map.update( + contract_name, + [entry], + &[entry | &1] + ) + else + acc + end + end) + |> Map.new(fn {contract_name, entries} -> + {contract_name, Enum.reverse(entries)} + end) + + epoch_rewards_events = + [@carbon_offsetting_fund_set_event_signature] + |> fetch_events_for_contract( + :epoch_rewards, + core_contract_addresses, + latest_block_number, + json_rpc_named_arguments + ) + |> Map.new(fn {address, logs} -> + entries = + logs + |> Enum.map( + &%{ + address: truncate_address_hash(&1.second_topic), + updated_at_block_number: &1.block_number + } + ) + + event_name = atom_to_contract_event_names[:epoch_rewards][:carbon_offsetting_fund_set] + {address, %{event_name => entries}} + end) + + fee_handler_events = + [ + @fee_beneficiary_set_event_signature, + @burn_fraction_set_event_signature + ] + |> fetch_events_for_contract( + :fee_handler, + core_contract_addresses, + latest_block_number, + json_rpc_named_arguments + ) + |> Map.new(fn {address, logs} -> + topic_to_logs = logs |> Enum.group_by(& &1.first_topic) + + fee_beneficiary_set_event_name = atom_to_contract_event_names[:fee_handler][:fee_beneficiary_set] + burn_fraction_set_event_name = atom_to_contract_event_names[:fee_handler][:burn_fraction_set] + + { + address, + %{ + fee_beneficiary_set_event_name => + topic_to_logs + |> Map.get(@fee_beneficiary_set_event_signature, []) + |> Enum.map( + &%{ + address: truncate_address_hash(&1.data), + updated_at_block_number: &1.block_number + } + ), + burn_fraction_set_event_name => + topic_to_logs + |> Map.get(@burn_fraction_set_event_signature, []) + |> Enum.map(fn log -> + [fraction] = decode_data(log.data, [{:int, 256}]) + + %{ + value: fraction, + updated_at_block_number: log.block_number + } + end) + } + } + end) + + core_contracts_json = + %{ + "addresses" => core_contract_addresses, + "events" => %{ + atom_to_contract_name[:epoch_rewards] => epoch_rewards_events, + atom_to_contract_name[:fee_handler] => fee_handler_events + } + } + |> Jason.encode!() + + IO.puts("CELO_CORE_CONTRACTS=#{core_contracts_json}") + end + + defp fetch_logs_by_chunks(from_block..to_block, requests_func, json_rpc_named_arguments) do + from_block..to_block + |> IndexerHelper.range_chunk_every(@chunk_size) + |> Enum.reduce([], fn chunk_start..chunk_end, acc -> + IndexerHelper.log_blocks_chunk_handling(chunk_start, chunk_end, 0, to_block, nil, :L1) + + requests = requests_func.(chunk_start, chunk_end) + + {:ok, responses} = + IndexerHelper.repeated_batch_rpc_call( + requests, + json_rpc_named_arguments, + fn message -> "Could not fetch logs: #{message}" end, + @max_request_retries + ) + + {:ok, logs} = Logs.from_responses(responses) + + acc ++ logs + end) + end + + defp fetch_events_for_contract( + event_signatures, + contract_atom, + core_contract_addresses, + latest_block_number, + json_rpc_named_arguments + ) do + contract_name = + CeloCoreContracts.atom_to_contract_name() + |> Map.get(contract_atom) + + core_contract_addresses + |> Map.get(contract_name, []) + |> Enum.chunk_every(2, 1) + |> Enum.map(fn + [entry, %{updated_at_block_number: to_block}] -> + {entry, to_block} + + [entry] -> + {entry, latest_block_number} + end) + |> Enum.map(fn {%{address: address}, to_block} -> + logs = + fetch_events_for_address( + 0..to_block, + event_signatures, + address, + json_rpc_named_arguments + ) + + {address, logs} + end) + end + + defp fetch_events_for_address(chunk_range, event_signatures, address, json_rpc_named_arguments) do + fetch_logs_by_chunks( + chunk_range, + fn chunk_start, chunk_end -> + event_signatures + |> Enum.with_index() + |> Enum.map(fn {signature, index} -> + Logs.request( + index, + %{ + from_block: chunk_start, + to_block: chunk_end, + address: address, + topics: [signature] + } + ) + end) + end, + json_rpc_named_arguments + ) + end +end diff --git a/apps/explorer/priv/celo/migrations/20240323152023_add_custom_fields.exs b/apps/explorer/priv/celo/migrations/20240323152023_add_custom_fields.exs new file mode 100644 index 0000000000..8de00c05e7 --- /dev/null +++ b/apps/explorer/priv/celo/migrations/20240323152023_add_custom_fields.exs @@ -0,0 +1,11 @@ +defmodule Explorer.Repo.Celo.Migrations.AddCustomFields do + use Ecto.Migration + + def change do + alter table(:transactions) do + add(:gateway_fee, :numeric, precision: 100, null: true) + add(:gas_token_contract_address_hash, :bytea, null: true) + add(:gas_fee_recipient_address_hash, :bytea, null: true) + end + end +end diff --git a/apps/explorer/priv/celo/migrations/20240424121856_add_pending_epoch_block_operations.exs b/apps/explorer/priv/celo/migrations/20240424121856_add_pending_epoch_block_operations.exs new file mode 100644 index 0000000000..3b36446be1 --- /dev/null +++ b/apps/explorer/priv/celo/migrations/20240424121856_add_pending_epoch_block_operations.exs @@ -0,0 +1,14 @@ +defmodule Explorer.Repo.Celo.Migrations.AddPendingEpochBlockOperations do + use Ecto.Migration + + def change do + create table(:celo_pending_epoch_block_operations, primary_key: false) do + add(:block_hash, references(:blocks, column: :hash, type: :bytea, on_delete: :delete_all), + null: false, + primary_key: true + ) + + timestamps() + end + end +end diff --git a/apps/explorer/priv/celo/migrations/20240512143204_remove_transaction_hash_from_primary_key_in_logs.exs b/apps/explorer/priv/celo/migrations/20240512143204_remove_transaction_hash_from_primary_key_in_logs.exs new file mode 100644 index 0000000000..9429c33f0e --- /dev/null +++ b/apps/explorer/priv/celo/migrations/20240512143204_remove_transaction_hash_from_primary_key_in_logs.exs @@ -0,0 +1,26 @@ +defmodule Explorer.Repo.Celo.Migrations.RemoveTransactionHashFromPrimaryKeyInLogs do + use Ecto.Migration + + def change do + execute( + """ + ALTER TABLE logs + DROP CONSTRAINT logs_pkey, + ADD PRIMARY KEY (block_hash, index); + """, + """ + ALTER TABLE logs + DROP CONSTRAINT logs_pkey, + ADD PRIMARY KEY (transaction_hash, block_hash, index); + """ + ) + + execute( + "ALTER TABLE logs ALTER COLUMN transaction_hash DROP NOT NULL", + "ALTER TABLE logs ALTER COLUMN transaction_hash SET NOT NULL" + ) + + drop(unique_index(:logs, [:transaction_hash, :index])) + create_if_not_exists(index(:logs, [:transaction_hash, :index])) + end +end diff --git a/apps/explorer/priv/celo/migrations/20240513091316_remove_transaction_hash_from_primary_key_in_token_transfers.exs b/apps/explorer/priv/celo/migrations/20240513091316_remove_transaction_hash_from_primary_key_in_token_transfers.exs new file mode 100644 index 0000000000..31dd75fb6e --- /dev/null +++ b/apps/explorer/priv/celo/migrations/20240513091316_remove_transaction_hash_from_primary_key_in_token_transfers.exs @@ -0,0 +1,23 @@ +defmodule Explorer.Repo.Celo.Migrations.RemoveTransactionHashFromPrimaryKeyInTokenTransfers do + use Ecto.Migration + + def change do + execute( + """ + ALTER TABLE token_transfers + DROP CONSTRAINT token_transfers_pkey, + ADD PRIMARY KEY (block_hash, log_index); + """, + """ + ALTER TABLE token_transfers + DROP CONSTRAINT token_transfers_pkey, + ADD PRIMARY KEY (transaction_hash, block_hash, log_index); + """ + ) + + execute( + "ALTER TABLE token_transfers ALTER COLUMN transaction_hash DROP NOT NULL", + "ALTER TABLE token_transfers ALTER COLUMN transaction_hash SET NOT NULL" + ) + end +end diff --git a/apps/explorer/priv/celo/migrations/20240607185817_add_epoch_rewards.exs b/apps/explorer/priv/celo/migrations/20240607185817_add_epoch_rewards.exs new file mode 100644 index 0000000000..b44e2069a2 --- /dev/null +++ b/apps/explorer/priv/celo/migrations/20240607185817_add_epoch_rewards.exs @@ -0,0 +1,62 @@ +defmodule Explorer.Repo.Celo.Migrations.AddEpochRewards do + use Ecto.Migration + + def change do + create table(:celo_epoch_rewards, primary_key: false) do + add(:reserve_bolster_transfer_log_index, :integer) + add(:community_transfer_log_index, :integer) + add(:carbon_offsetting_transfer_log_index, :integer) + + add( + :block_hash, + references(:blocks, column: :hash, type: :bytea, on_delete: :delete_all), + null: false, + primary_key: true + ) + + timestamps() + end + + execute( + """ + ALTER TABLE celo_epoch_rewards + ADD CONSTRAINT celo_epoch_rewards_reserve_bolster_transfer_log_index_fkey + FOREIGN KEY (reserve_bolster_transfer_log_index, block_hash) + REFERENCES token_transfers (log_index, block_hash) + ON DELETE CASCADE + """, + """ + ALTER TABLE celo_epoch_rewards + DROP CONSTRAINT celo_epoch_rewards_reserve_bolster_transfer_log_index_fkey + """ + ) + + execute( + """ + ALTER TABLE celo_epoch_rewards + ADD CONSTRAINT celo_epoch_rewards_community_transfer_log_index_fkey + FOREIGN KEY (community_transfer_log_index, block_hash) + REFERENCES token_transfers (log_index, block_hash) + ON DELETE CASCADE + """, + """ + ALTER TABLE celo_epoch_rewards + DROP CONSTRAINT celo_epoch_rewards_community_transfer_log_index_fkey + """ + ) + + execute( + """ + ALTER TABLE celo_epoch_rewards + ADD CONSTRAINT celo_epoch_rewards_carbon_offsetting_transfer_log_index_fkey + FOREIGN KEY (carbon_offsetting_transfer_log_index, block_hash) + REFERENCES token_transfers (log_index, block_hash) + ON DELETE CASCADE + """, + """ + ALTER TABLE celo_epoch_rewards + DROP CONSTRAINT celo_epoch_rewards_carbon_offsetting_transfer_log_index_fkey + """ + ) + end +end diff --git a/apps/explorer/priv/celo/migrations/20240612135216_add_validator_group_votes.exs b/apps/explorer/priv/celo/migrations/20240612135216_add_validator_group_votes.exs new file mode 100644 index 0000000000..3d6f1f4eeb --- /dev/null +++ b/apps/explorer/priv/celo/migrations/20240612135216_add_validator_group_votes.exs @@ -0,0 +1,36 @@ +defmodule Explorer.Repo.Celo.Migrations.AddValidatorGroupVotes do + use Ecto.Migration + + def change do + execute( + "CREATE TYPE celo_validator_group_vote_type AS ENUM ('activated', 'revoked')", + "DROP TYPE celo_validator_group_vote_type" + ) + + create table(:celo_validator_group_votes, primary_key: false) do + add( + :account_address_hash, + references(:addresses, column: :hash, on_delete: :delete_all, type: :bytea), + null: false + ) + + add( + :group_address_hash, + references(:addresses, column: :hash, on_delete: :delete_all, type: :bytea), + null: false + ) + + add(:value, :numeric, precision: 100, null: false) + add(:units, :numeric, precision: 100, null: false) + + add(:type, :celo_validator_group_vote_type, null: false) + + add(:transaction_hash, :bytea, null: false, primary_key: true) + + add(:block_number, :integer, null: false) + add(:block_hash, :bytea, null: false) + + timestamps() + end + end +end diff --git a/apps/explorer/priv/celo/migrations/20240614125614_add_election_rewards.exs b/apps/explorer/priv/celo/migrations/20240614125614_add_election_rewards.exs new file mode 100644 index 0000000000..30dc3a67de --- /dev/null +++ b/apps/explorer/priv/celo/migrations/20240614125614_add_election_rewards.exs @@ -0,0 +1,38 @@ +defmodule Explorer.Repo.Celo.Migrations.AddElectionRewards do + use Ecto.Migration + + def change do + execute( + "CREATE TYPE celo_election_reward_type AS ENUM ('voter', 'validator', 'group', 'delegated_payment')", + "DROP TYPE celo_election_reward_type" + ) + + create table(:celo_election_rewards, primary_key: false) do + add(:amount, :numeric, precision: 100, null: false) + add(:type, :celo_election_reward_type, null: false, primary_key: true) + + add( + :block_hash, + references(:blocks, column: :hash, type: :bytea, on_delete: :delete_all), + null: false, + primary_key: true + ) + + add( + :account_address_hash, + references(:addresses, column: :hash, on_delete: :delete_all, type: :bytea), + null: false, + primary_key: true + ) + + add( + :associated_account_address_hash, + references(:addresses, column: :hash, on_delete: :delete_all, type: :bytea), + null: false, + primary_key: true + ) + + timestamps() + end + end +end diff --git a/apps/explorer/priv/celo/migrations/20240715110334_remove_unused_fields_from_validator_group_votes.exs b/apps/explorer/priv/celo/migrations/20240715110334_remove_unused_fields_from_validator_group_votes.exs new file mode 100644 index 0000000000..8d678d5c83 --- /dev/null +++ b/apps/explorer/priv/celo/migrations/20240715110334_remove_unused_fields_from_validator_group_votes.exs @@ -0,0 +1,10 @@ +defmodule Explorer.Repo.Celo.Migrations.RemoveUnusedFieldsFromValidatorGroupVotes do + use Ecto.Migration + + def change do + alter table(:celo_validator_group_votes) do + remove(:value, :numeric, precision: 100, null: false, default: 0) + remove(:units, :numeric, precision: 100, null: false, default: 0) + end + end +end diff --git a/apps/explorer/test/explorer/chain/cache/celo_core_contracts_test.exs b/apps/explorer/test/explorer/chain/cache/celo_core_contracts_test.exs new file mode 100644 index 0000000000..6a3284e462 --- /dev/null +++ b/apps/explorer/test/explorer/chain/cache/celo_core_contracts_test.exs @@ -0,0 +1,84 @@ +defmodule Explorer.Chain.Cache.CeloCoreContractsTest do + use ExUnit.Case, async: true + + alias Explorer.Chain.Cache.CeloCoreContracts + + describe "get_address/2" do + test "returns address according to block number" do + first_address = "0xb10ee11244526b94879e1956745ba2e35ae2ba20" + + Application.put_env(:explorer, Explorer.Chain.Cache.CeloCoreContracts, + contracts: %{ + "addresses" => %{ + "EpochRewards" => [ + %{ + "address" => first_address, + "updated_at_block_number" => 100 + } + ] + } + } + ) + + on_exit(fn -> + Application.put_env(:explorer, Explorer.Chain.Cache.CeloCoreContracts, contracts: %{}) + end) + + assert {:error, :address_does_not_exist} = CeloCoreContracts.get_address(:epoch_rewards, 99) + assert {:ok, ^first_address} = CeloCoreContracts.get_address(:epoch_rewards, 100) + assert {:ok, ^first_address} = CeloCoreContracts.get_address(:epoch_rewards, 10_000) + end + end + + describe "get_event/3" do + test "returns event according to block number" do + first_address = "0x0000000000000000000000000000000000000000" + second_address = "0x22579ca45ee22e2e16ddf72d955d6cf4c767b0ef" + + Application.put_env(:explorer, Explorer.Chain.Cache.CeloCoreContracts, + contracts: %{ + "addresses" => %{ + "EpochRewards" => [ + %{ + "address" => "0xb10ee11244526b94879e1956745ba2e35ae2ba20", + "updated_at_block_number" => 100 + } + ] + }, + "events" => %{ + "EpochRewards" => %{ + "0xb10ee11244526b94879e1956745ba2e35ae2ba20" => %{ + "CarbonOffsettingFundSet" => [ + %{ + "address" => first_address, + "updated_at_block_number" => 598 + }, + %{ + "address" => second_address, + "updated_at_block_number" => 15_049_265 + } + ] + } + } + } + } + ) + + on_exit(fn -> + Application.put_env(:explorer, Explorer.Chain.Cache.CeloCoreContracts, contracts: %{}) + end) + + assert {:error, :address_does_not_exist} = + CeloCoreContracts.get_event(:epoch_rewards, :carbon_offsetting_fund_set, 99) + + assert {:ok, %{"address" => ^first_address, "updated_at_block_number" => 598}} = + CeloCoreContracts.get_event(:epoch_rewards, :carbon_offsetting_fund_set, 598) + + assert {:ok, %{"address" => ^first_address, "updated_at_block_number" => 598}} = + CeloCoreContracts.get_event(:epoch_rewards, :carbon_offsetting_fund_set, 599) + + assert {:ok, %{"address" => ^second_address, "updated_at_block_number" => 15_049_265}} = + CeloCoreContracts.get_event(:epoch_rewards, :carbon_offsetting_fund_set, 16_000_000) + end + end +end diff --git a/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs b/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs index 9c5107f98b..2ba66c3464 100644 --- a/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs +++ b/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs @@ -8,9 +8,12 @@ defmodule Explorer.Chain.Import.Runner.BlocksTest do alias Ecto.Multi alias Explorer.Chain.Import.Runner.{Blocks, Transactions} alias Explorer.Chain.{Address, Block, Transaction, PendingBlockOperation} + alias Explorer.Chain.Celo.PendingEpochBlockOperation alias Explorer.{Chain, Repo} alias Explorer.Utility.MissingBlockRange + alias Explorer.Chain.Celo.Helper, as: CeloHelper + describe "run/1" do setup do miner = insert(:address) @@ -411,6 +414,60 @@ defmodule Explorer.Chain.Import.Runner.BlocksTest do assert %{block_number: ^number, block_hash: ^hash} = Repo.one(PendingBlockOperation) end + if Application.compile_env(:explorer, :chain_type) == :celo do + test "inserts pending_epoch_block_operations only for epoch blocks", + %{consensus_block: %{miner_hash: miner_hash}, options: options} do + epoch_block_number = CeloHelper.blocks_per_epoch() + + %{hash: hash} = + epoch_block_params = + params_for( + :block, + miner_hash: miner_hash, + consensus: true, + number: epoch_block_number + ) + + non_epoch_block_params = + params_for( + :block, + miner_hash: miner_hash, + consensus: true, + number: epoch_block_number + 1 + ) + + insert_block(epoch_block_params, options) + insert_block(non_epoch_block_params, options) + + assert %{block_hash: ^hash} = Repo.one(PendingEpochBlockOperation) + end + + test "inserts pending_epoch_block_operations only for consensus epoch blocks", + %{consensus_block: %{miner_hash: miner_hash}, options: options} do + %{hash: hash} = + first_epoch_block_params = + params_for( + :block, + miner_hash: miner_hash, + consensus: true, + number: CeloHelper.blocks_per_epoch() + ) + + second_epoch_block_params = + params_for( + :block, + miner_hash: miner_hash, + consensus: false, + number: CeloHelper.blocks_per_epoch() * 2 + ) + + insert_block(first_epoch_block_params, options) + insert_block(second_epoch_block_params, options) + + assert %{block_hash: ^hash} = Repo.one(PendingEpochBlockOperation) + end + end + test "change instance owner if was token transfer in older blocks", %{consensus_block: %{hash: block_hash, miner_hash: miner_hash, number: block_number}, options: options} do block_number = block_number + 2 diff --git a/apps/explorer/test/support/factory.ex b/apps/explorer/test/support/factory.ex index 659b40f252..4c090b6954 100644 --- a/apps/explorer/test/support/factory.ex +++ b/apps/explorer/test/support/factory.ex @@ -24,6 +24,7 @@ defmodule Explorer.Factory do alias Explorer.Chain.Beacon.{Blob, BlobTransaction} alias Explorer.Chain.Block.{EmissionReward, Range, Reward} alias Explorer.Chain.Stability.Validator, as: ValidatorStability + alias Explorer.Chain.Celo.PendingEpochBlockOperation, as: CeloPendingEpochBlockOperation alias Explorer.Chain.{ Address, @@ -1254,4 +1255,8 @@ defmodule Explorer.Factory do state: Enum.random(0..2) } end + + def celo_pending_epoch_block_operation_factory do + %CeloPendingEpochBlockOperation{} + end end diff --git a/apps/indexer/lib/indexer/block/catchup/fetcher.ex b/apps/indexer/lib/indexer/block/catchup/fetcher.ex index 67ab266f98..180ddf37ec 100644 --- a/apps/indexer/lib/indexer/block/catchup/fetcher.ex +++ b/apps/indexer/lib/indexer/block/catchup/fetcher.ex @@ -11,13 +11,14 @@ defmodule Indexer.Block.Catchup.Fetcher do only: [ async_import_blobs: 2, async_import_block_rewards: 2, + async_import_celo_epoch_block_operations: 2, async_import_coin_balances: 2, async_import_created_contract_codes: 2, async_import_internal_transactions: 2, async_import_replaced_transactions: 2, - async_import_tokens: 2, async_import_token_balances: 2, async_import_token_instances: 1, + async_import_tokens: 2, async_import_uncles: 2, fetch_and_import_range: 2 ] @@ -139,6 +140,7 @@ defmodule Indexer.Block.Catchup.Fetcher do async_import_replaced_transactions(imported, realtime?) async_import_token_instances(imported) async_import_blobs(imported, realtime?) + async_import_celo_epoch_block_operations(imported, realtime?) end defp stream_fetch_and_import(state, ranges) do diff --git a/apps/indexer/lib/indexer/block/fetcher.ex b/apps/indexer/lib/indexer/block/fetcher.ex index ff0a342881..03082037a2 100644 --- a/apps/indexer/lib/indexer/block/fetcher.ex +++ b/apps/indexer/lib/indexer/block/fetcher.ex @@ -16,11 +16,15 @@ defmodule Indexer.Block.Fetcher do alias Explorer.Chain.Cache.Blocks, as: BlocksCache alias Explorer.Chain.Cache.{Accounts, BlockNumber, Transactions, Uncles} alias Indexer.Block.Fetcher.Receipts + alias Indexer.Fetcher.Celo.EpochBlockOperations, as: CeloEpochBlockOperations + alias Indexer.Fetcher.Celo.EpochLogs, as: CeloEpochLogs alias Indexer.Fetcher.CoinBalance.Catchup, as: CoinBalanceCatchup alias Indexer.Fetcher.CoinBalance.Realtime, as: CoinBalanceRealtime alias Indexer.Fetcher.PolygonZkevm.BridgeL1Tokens, as: PolygonZkevmBridgeL1Tokens alias Indexer.Fetcher.TokenInstance.Realtime, as: TokenInstanceRealtime + alias Indexer.{Prometheus, TokenBalances, Tracer} + alias Indexer.Fetcher.{ Beacon.Blob, BlockReward, @@ -32,8 +36,6 @@ defmodule Indexer.Block.Fetcher do UncleBlock } - alias Indexer.{Prometheus, TokenBalances, Tracer} - alias Indexer.Transform.{ AddressCoinBalances, Addresses, @@ -54,6 +56,9 @@ defmodule Indexer.Block.Fetcher do alias Indexer.Transform.Blocks, as: TransformBlocks alias Indexer.Transform.PolygonZkevm.Bridge, as: PolygonZkevmBridge + alias Indexer.Transform.Celo.TransactionGasTokens, as: CeloTransactionGasTokens + alias Indexer.Transform.Celo.TransactionTokenTransfers, as: CeloTransactionTokenTransfers + @type address_hash_to_fetched_balance_block_number :: %{String.t() => Block.block_number()} @type t :: %__MODULE__{} @@ -148,9 +153,16 @@ defmodule Indexer.Block.Fetcher do }}} <- {:blocks, fetched_blocks}, blocks = TransformBlocks.transform_blocks(blocks_params), {:receipts, {:ok, receipt_params}} <- {:receipts, Receipts.fetch(state, transactions_params_without_receipts)}, - %{logs: logs, receipts: receipts} = receipt_params, + %{logs: receipt_logs, receipts: receipts} = receipt_params, transactions_with_receipts = Receipts.put(transactions_params_without_receipts, receipts), + celo_epoch_logs = CeloEpochLogs.fetch(blocks, json_rpc_named_arguments), + logs = receipt_logs ++ celo_epoch_logs, %{token_transfers: token_transfers, tokens: tokens} = TokenTransfers.parse(logs), + %{token_transfers: celo_native_token_transfers, tokens: celo_tokens} = + CeloTransactionTokenTransfers.parse_transactions(transactions_with_receipts), + celo_gas_tokens = CeloTransactionGasTokens.parse(transactions_with_receipts), + token_transfers = token_transfers ++ celo_native_token_transfers, + tokens = Enum.uniq(tokens ++ celo_tokens), %{transaction_actions: transaction_actions} = TransactionActions.parse(logs), %{mint_transfers: mint_transfers} = MintTransfers.parse(logs), optimism_withdrawals = @@ -229,6 +241,7 @@ defmodule Indexer.Block.Fetcher do polygon_edge_deposit_executes: polygon_edge_deposit_executes, polygon_zkevm_bridge_operations: polygon_zkevm_bridge_operations, shibarium_bridge_operations: shibarium_bridge_operations, + celo_gas_tokens: celo_gas_tokens, arbitrum_messages: arbitrum_xlevel_messages }, {:ok, inserted} <- @@ -264,6 +277,7 @@ defmodule Indexer.Block.Fetcher do polygon_edge_deposit_executes: polygon_edge_deposit_executes, polygon_zkevm_bridge_operations: polygon_zkevm_bridge_operations, shibarium_bridge_operations: shibarium_bridge_operations, + celo_gas_tokens: celo_gas_tokens, arbitrum_messages: arbitrum_xlevel_messages }) do case Application.get_env(:explorer, :chain_type) do @@ -290,6 +304,18 @@ defmodule Indexer.Block.Fetcher do basic_import_options |> Map.put_new(:shibarium_bridge_operations, %{params: shibarium_bridge_operations}) + :celo -> + tokens = + basic_import_options + |> Map.get(:tokens, %{}) + |> Map.get(:params, []) + + basic_import_options + |> Map.put( + :tokens, + %{params: (tokens ++ celo_gas_tokens) |> Enum.uniq()} + ) + :arbitrum -> basic_import_options |> Map.put_new(:arbitrum_messages, %{params: arbitrum_xlevel_messages}) @@ -477,6 +503,14 @@ defmodule Indexer.Block.Fetcher do def async_import_polygon_zkevm_bridge_l1_tokens(_), do: :ok + def async_import_celo_epoch_block_operations(%{blocks: operations}, realtime?) do + operations + |> Enum.map(&%{block_number: &1.number, block_hash: &1.hash}) + |> CeloEpochBlockOperations.async_fetch(realtime?) + end + + def async_import_celo_epoch_block_operations(_, _), do: :ok + defp block_reward_errors_to_block_numbers(block_reward_errors) when is_list(block_reward_errors) do Enum.map(block_reward_errors, &block_reward_error_to_block_number/1) end @@ -685,7 +719,7 @@ defmodule Indexer.Block.Fetcher do Map.delete(address_params, :fetched_coin_balance_block_number)} end - defp token_transfers_merge_token(token_transfers, tokens) do + def token_transfers_merge_token(token_transfers, tokens) do Enum.map(token_transfers, fn token_transfer -> token = Enum.find(tokens, fn token -> diff --git a/apps/indexer/lib/indexer/block/realtime/fetcher.ex b/apps/indexer/lib/indexer/block/realtime/fetcher.ex index dc51dce76a..10d6fbafc2 100644 --- a/apps/indexer/lib/indexer/block/realtime/fetcher.ex +++ b/apps/indexer/lib/indexer/block/realtime/fetcher.ex @@ -13,17 +13,18 @@ defmodule Indexer.Block.Realtime.Fetcher do import Indexer.Block.Fetcher, only: [ - async_import_realtime_coin_balances: 1, async_import_blobs: 2, async_import_block_rewards: 2, + async_import_celo_epoch_block_operations: 2, async_import_created_contract_codes: 2, async_import_internal_transactions: 2, + async_import_polygon_zkevm_bridge_l1_tokens: 1, + async_import_realtime_coin_balances: 1, async_import_replaced_transactions: 2, - async_import_tokens: 2, async_import_token_balances: 2, async_import_token_instances: 1, + async_import_tokens: 2, async_import_uncles: 2, - async_import_polygon_zkevm_bridge_l1_tokens: 1, fetch_and_import_range: 2 ] @@ -465,5 +466,6 @@ defmodule Indexer.Block.Realtime.Fetcher do async_import_replaced_transactions(imported, realtime?) async_import_blobs(imported, realtime?) async_import_polygon_zkevm_bridge_l1_tokens(imported) + async_import_celo_epoch_block_operations(imported, realtime?) end end diff --git a/apps/indexer/lib/indexer/fetcher/celo/epoch_block_operations.ex b/apps/indexer/lib/indexer/fetcher/celo/epoch_block_operations.ex new file mode 100644 index 0000000000..e023ac2bb9 --- /dev/null +++ b/apps/indexer/lib/indexer/fetcher/celo/epoch_block_operations.ex @@ -0,0 +1,146 @@ +defmodule Indexer.Fetcher.Celo.EpochBlockOperations do + @moduledoc """ + Tracks epoch blocks awaiting processing by the epoch fetcher. + """ + + import Explorer.Chain.Celo.Helper, only: [epoch_block_number?: 1] + + alias Explorer.Chain + alias Explorer.Chain.Block + alias Explorer.Chain.Celo.PendingEpochBlockOperation + alias Indexer.Fetcher.Celo.EpochBlockOperations.Supervisor, as: EpochBlockOperationsSupervisor + alias Indexer.Transform.Addresses + alias Indexer.{BufferedTask, Tracer} + + alias Indexer.Fetcher.Celo.EpochBlockOperations.{ + DelegatedPayments, + Distributions, + ValidatorAndGroupPayments, + VoterPayments + } + + require Logger + + use Indexer.Fetcher, restart: :permanent + use Spandex.Decorators + + @behaviour BufferedTask + + @default_max_batch_size 1 + @default_max_concurrency 1 + + @doc false + def child_spec([init_options, gen_server_options]) do + {state, mergeable_init_options} = Keyword.pop(init_options, :json_rpc_named_arguments) + + unless state do + raise ArgumentError, + ":json_rpc_named_arguments must be provided to `#{__MODULE__}.child_spec " <> + "to allow for json_rpc calls when running." + end + + merged_init_opts = + defaults() + |> Keyword.merge(mergeable_init_options) + |> Keyword.put(:state, state) + + Supervisor.child_spec({BufferedTask, [{__MODULE__, merged_init_opts}, gen_server_options]}, id: __MODULE__) + end + + def defaults do + [ + poll: false, + flush_interval: :timer.seconds(3), + max_concurrency: Application.get_env(:indexer, __MODULE__)[:concurrency] || @default_max_concurrency, + max_batch_size: Application.get_env(:indexer, __MODULE__)[:batch_size] || @default_max_batch_size, + task_supervisor: Indexer.Fetcher.Celo.EpochBlockOperations.TaskSupervisor, + metadata: [fetcher: :celo_epoch_rewards] + ] + end + + @spec async_fetch( + [%{block_number: Block.block_number(), block_hash: Hash.Full}], + boolean(), + integer() + ) :: :ok + def async_fetch(entries, realtime?, timeout \\ 5000) when is_list(entries) do + if EpochBlockOperationsSupervisor.disabled?() do + :ok + else + filtered_entries = Enum.filter(entries, &epoch_block_number?(&1.block_number)) + BufferedTask.buffer(__MODULE__, filtered_entries, realtime?, timeout) + end + end + + @impl BufferedTask + def init(initial, reducer, _json_rpc_named_arguments) do + {:ok, final} = + PendingEpochBlockOperation.stream_epoch_blocks_with_unfetched_rewards( + initial, + reducer, + true + ) + + final + end + + @impl BufferedTask + @decorate trace( + name: "fetch", + resource: "Indexer.Fetcher.Celo.EpochBlockOperations.run/2", + service: :indexer, + tracer: Tracer + ) + def run(pending_operations, json_rpc_named_arguments) do + Enum.each(pending_operations, fn pending_operation -> + fetch(pending_operation, json_rpc_named_arguments) + end) + + :ok + end + + defp fetch(pending_operation, json_rpc_named_arguments) do + {:ok, distributions} = Distributions.fetch(pending_operation) + {:ok, validator_and_group_payments} = ValidatorAndGroupPayments.fetch(pending_operation) + + {:ok, voter_payments} = + VoterPayments.fetch( + pending_operation, + json_rpc_named_arguments + ) + + {:ok, delegated_payments} = + validator_and_group_payments + |> Enum.filter(&(&1.type == :validator)) + |> Enum.map(& &1.account_address_hash) + |> DelegatedPayments.fetch( + pending_operation, + json_rpc_named_arguments + ) + + election_rewards = + [ + validator_and_group_payments, + voter_payments, + delegated_payments + ] + |> Enum.concat() + |> Enum.filter(&(&1.amount > 0)) + + addresses_params = + Addresses.extract_addresses(%{ + celo_election_rewards: election_rewards + }) + + {:ok, imported} = + Chain.import(%{ + addresses: %{params: addresses_params}, + celo_election_rewards: %{params: election_rewards}, + celo_epoch_rewards: %{params: [distributions]} + }) + + Logger.info("Fetched epoch rewards for block number: #{pending_operation.block_number}") + + {:ok, imported} + end +end diff --git a/apps/indexer/lib/indexer/fetcher/celo/epoch_block_operations/core_contract_version.ex b/apps/indexer/lib/indexer/fetcher/celo/epoch_block_operations/core_contract_version.ex new file mode 100644 index 0000000000..d6c011de9b --- /dev/null +++ b/apps/indexer/lib/indexer/fetcher/celo/epoch_block_operations/core_contract_version.ex @@ -0,0 +1,92 @@ +defmodule Indexer.Fetcher.Celo.EpochBlockOperations.CoreContractVersion do + @moduledoc """ + Fetches the version of the celo core contract. + """ + import Indexer.Fetcher.Celo.Helper, only: [abi_to_method_id: 1] + import Indexer.Helper, only: [read_contracts_with_retries: 5] + + @repeated_request_max_retries 3 + + @get_version_number_abi [ + %{ + "name" => "getVersionNumber", + "type" => "function", + "payable" => false, + "constant" => true, + "stateMutability" => "pure", + "inputs" => [], + "outputs" => [ + %{"type" => "uint256"}, + %{"type" => "uint256"}, + %{"type" => "uint256"}, + %{"type" => "uint256"} + ] + } + ] + + @get_version_number_method_id @get_version_number_abi |> abi_to_method_id() + + @doc """ + Fetches the version number of a Celo core contract at a given block. + + ## Parameters + - `contract_address` (`EthereumJSONRPC.address()`): The address of the + contract. + - `block_number` (`EthereumJSONRPC.block_number()`): The block number at + which to fetch the version. + - `json_rpc_named_arguments` (`EthereumJSONRPC.json_rpc_named_arguments()`): + The JSON RPC named arguments. + + ## Returns + - `{:ok, {integer(), integer(), integer(), integer()}}`: A tuple containing + the version number components if successful. + - `{:ok, {1, 1, 0, 0}}`: A default version number if the `getVersionNumber` + function does not exist for the core contract at the requested block. + - `{:error, [{any(), any()}]}`: An error tuple with the list of errors. + """ + @spec fetch( + EthereumJSONRPC.address(), + EthereumJSONRPC.block_number(), + EthereumJSONRPC.json_rpc_named_arguments() + ) :: + { + :error, + [{any(), any()}] + } + | { + :ok, + {integer(), integer(), integer(), integer()} + } + def fetch(contract_address, block_number, json_rpc_named_arguments) do + request = %{ + contract_address: contract_address, + method_id: @get_version_number_method_id, + args: [], + block_number: block_number + } + + [request] + |> read_contracts_with_retries( + @get_version_number_abi, + json_rpc_named_arguments, + @repeated_request_max_retries, + false + ) + |> elem(0) + |> case do + [ok: [storage, major, minor, patch]] -> + {:ok, {storage, major, minor, patch}} + + # Celo Core Contracts deployed to a live network without the + # `getVersionNumber()` function, such as the original set of core + # contracts, are to be considered version 1.1.0.0. + # + # https://docs.celo.org/community/release-process/smart-contracts#core-contracts + [error: "(-32000) execution reverted"] -> + {:ok, {1, 1, 0, 0}} + + errors -> + {:error, errors} + end + end +end diff --git a/apps/indexer/lib/indexer/fetcher/celo/epoch_block_operations/delegated_payments.ex b/apps/indexer/lib/indexer/fetcher/celo/epoch_block_operations/delegated_payments.ex new file mode 100644 index 0000000000..8569019a3b --- /dev/null +++ b/apps/indexer/lib/indexer/fetcher/celo/epoch_block_operations/delegated_payments.ex @@ -0,0 +1,166 @@ +defmodule Indexer.Fetcher.Celo.EpochBlockOperations.DelegatedPayments do + @moduledoc """ + Fetches delegated validator payments for the epoch block. + """ + import Ecto.Query, only: [from: 2] + import Explorer.Chain.SmartContract, only: [burn_address_hash_string: 0] + import Indexer.Fetcher.Celo.Helper, only: [abi_to_method_id: 1] + import Indexer.Helper, only: [read_contracts_with_retries: 4] + + alias Explorer.Chain.Cache.CeloCoreContracts + alias Explorer.Chain.{Hash, TokenTransfer} + alias Explorer.Repo + alias Indexer.Fetcher.Celo.EpochBlockOperations.CoreContractVersion + + require Logger + + @mint_address_hash_string burn_address_hash_string() + + @repeated_request_max_retries 3 + + # The method `getPaymentDelegation` was introduced in the following. Thus, we + # set version hardcoded in `getVersionNumber` method. + # + # https://github.com/celo-org/celo-monorepo/blob/d7c8936dc529f46d56799365f8b3383a23cc220b/packages/protocol/contracts/common/Accounts.sol#L128-L130 + @get_payment_delegation_available_since_version {1, 1, 3, 0} + @get_payment_delegation_abi [ + %{ + "name" => "getPaymentDelegation", + "type" => "function", + "payable" => false, + "constant" => true, + "stateMutability" => "view", + "inputs" => [ + %{"name" => "account", "type" => "address"} + ], + "outputs" => [ + %{"type" => "address"}, + %{"type" => "uint256"} + ] + } + ] + @get_payment_delegation_method_id @get_payment_delegation_abi |> abi_to_method_id() + + @spec fetch( + [EthereumJSONRPC.address()], + %{ + :block_hash => EthereumJSONRPC.hash(), + :block_number => EthereumJSONRPC.block_number() + }, + EthereumJSONRPC.json_rpc_named_arguments() + ) :: + {:ok, list()} + | {:error, any()} + def fetch( + validator_addresses, + %{block_number: block_number, block_hash: block_hash} = _pending_operation, + json_rpc_named_arguments + ) do + with {:ok, accounts_contract_address} <- + CeloCoreContracts.get_address(:accounts, block_number), + {:ok, accounts_contract_version} <- + CoreContractVersion.fetch( + accounts_contract_address, + block_number, + json_rpc_named_arguments + ), + true <- accounts_contract_version >= @get_payment_delegation_available_since_version, + {:ok, usd_token_contract_address} <- + CeloCoreContracts.get_address(:usd_token, block_number), + {responses, []} <- + read_payment_delegations( + validator_addresses, + accounts_contract_address, + block_number, + json_rpc_named_arguments + ) do + query = + from( + tt in TokenTransfer.only_consensus_transfers_query(), + where: + tt.block_hash == ^block_hash and + tt.token_contract_address_hash == ^usd_token_contract_address and + tt.from_address_hash == ^@mint_address_hash_string and + is_nil(tt.transaction_hash), + select: {tt.to_address_hash, tt.amount} + ) + + beneficiary_address_to_amount = + query + |> Repo.all() + |> Map.new(fn {address, amount} -> + {Hash.to_string(address), amount} + end) + + rewards = + validator_addresses + |> Enum.zip(responses) + |> Enum.filter(&match?({_, {:ok, [_, fraction]}} when fraction > 0, &1)) + |> Enum.map(fn + {validator_address, {:ok, [beneficiary_address, _]}} -> + amount = Map.get(beneficiary_address_to_amount, beneficiary_address, 0) + + %{ + block_hash: block_hash, + account_address_hash: beneficiary_address, + amount: amount, + associated_account_address_hash: validator_address, + type: :delegated_payment + } + end) + + {:ok, rewards} + else + false -> + Logger.info(fn -> + [ + "Do not fetch payment delegations since `getPaymentDelegation` ", + "method is not available on block #{block_number}" + ] + end) + + {:ok, []} + + {_, ["(-32000) execution reverted"]} -> + # todo: we should start fetching payment delegations only after the + # first `PaymentDelegationSet` event is emitted. Unfortunately, relying + # on contract version is not enough since the method could not be + # present. + Logger.info(fn -> + [ + "Could not fetch payment delegations since `getPaymentDelegation` constantly returns error. ", + "Most likely, the method is not available on block #{block_number}. " + ] + end) + + {:ok, []} + + error -> + Logger.error("Could not fetch payment delegations: #{inspect(error)}") + + error + end + end + + defp read_payment_delegations( + validator_addresses, + accounts_contract_address, + block_number, + json_rpc_named_arguments + ) do + validator_addresses + |> Enum.map( + &%{ + contract_address: accounts_contract_address, + method_id: @get_payment_delegation_method_id, + args: [&1], + block_number: block_number + } + ) + |> read_contracts_with_retries( + @get_payment_delegation_abi, + json_rpc_named_arguments, + @repeated_request_max_retries + ) + end +end diff --git a/apps/indexer/lib/indexer/fetcher/celo/epoch_block_operations/distributions.ex b/apps/indexer/lib/indexer/fetcher/celo/epoch_block_operations/distributions.ex new file mode 100644 index 0000000000..e21ce3f535 --- /dev/null +++ b/apps/indexer/lib/indexer/fetcher/celo/epoch_block_operations/distributions.ex @@ -0,0 +1,101 @@ +defmodule Indexer.Fetcher.Celo.EpochBlockOperations.Distributions do + @moduledoc """ + Fetches Reserve bolster, Community, and Carbon offsetting distributions for + the epoch block. + """ + import Ecto.Query, only: [from: 2, subquery: 1] + import Explorer.Chain.SmartContract, only: [burn_address_hash_string: 0] + + alias Explorer.Repo + + alias Explorer.Chain.{ + Cache.CeloCoreContracts, + Hash, + TokenTransfer + } + + @mint_address_hash_string burn_address_hash_string() + + @spec fetch(%{ + :block_hash => EthereumJSONRPC.hash(), + :block_number => EthereumJSONRPC.block_number() + }) :: + {:ok, map()} + | {:error, :multiple_transfers_to_same_address} + def fetch(%{block_number: block_number, block_hash: block_hash} = _pending_operation) do + {:ok, celo_token_contract_address_hash} = CeloCoreContracts.get_address(:celo_token, block_number) + {:ok, reserve_contract_address_hash} = CeloCoreContracts.get_address(:reserve, block_number) + {:ok, community_contract_address_hash} = CeloCoreContracts.get_address(:governance, block_number) + + {:ok, %{"address" => carbon_offsetting_contract_address_hash}} = + CeloCoreContracts.get_event(:epoch_rewards, :carbon_offsetting_fund_set, block_number) + + celo_mint_transfers_query = + from( + tt in TokenTransfer.only_consensus_transfers_query(), + where: + tt.block_hash == ^block_hash and + tt.token_contract_address_hash == ^celo_token_contract_address_hash and + tt.from_address_hash == ^@mint_address_hash_string and + is_nil(tt.transaction_hash) + ) + + # Every epoch has at least one CELO transfer from the zero address to the + # reserve. This is how cUSD is minted before it is distributed to + # validators. If there is only one CELO transfer, then there was no + # Reserve bolster distribution for that epoch. If there are multiple CELO + # transfers, then the last one is the Reserve bolster distribution. + reserve_bolster_transfer_log_index_query = + from( + tt in subquery( + from( + tt in subquery(celo_mint_transfers_query), + where: tt.to_address_hash == ^reserve_contract_address_hash, + order_by: tt.log_index, + offset: 1 + ) + ), + select: max(tt.log_index) + ) + + query = + from( + tt in subquery(celo_mint_transfers_query), + where: + tt.to_address_hash in ^[ + community_contract_address_hash, + carbon_offsetting_contract_address_hash + ] or + tt.log_index == subquery(reserve_bolster_transfer_log_index_query), + select: {tt.to_address_hash, tt.log_index} + ) + + transfers_with_log_index = query |> Repo.all() + + unique_addresses_count = + transfers_with_log_index + |> Enum.map(&elem(&1, 0)) + |> Enum.uniq() + |> Enum.count() + + address_to_key = %{ + reserve_contract_address_hash => :reserve_bolster_transfer_log_index, + community_contract_address_hash => :community_transfer_log_index, + carbon_offsetting_contract_address_hash => :carbon_offsetting_transfer_log_index + } + + if unique_addresses_count == Enum.count(transfers_with_log_index) do + distributions = + transfers_with_log_index + |> Enum.reduce(%{}, fn {address, log_index}, acc -> + key = Map.get(address_to_key, address |> Hash.to_string()) + Map.put(acc, key, log_index) + end) + |> Map.put(:block_hash, block_hash) + + {:ok, distributions} + else + {:error, :multiple_transfers_to_same_address} + end + end +end diff --git a/apps/indexer/lib/indexer/fetcher/celo/epoch_block_operations/validator_and_group_payments.ex b/apps/indexer/lib/indexer/fetcher/celo/epoch_block_operations/validator_and_group_payments.ex new file mode 100644 index 0000000000..f677d74394 --- /dev/null +++ b/apps/indexer/lib/indexer/fetcher/celo/epoch_block_operations/validator_and_group_payments.ex @@ -0,0 +1,67 @@ +defmodule Indexer.Fetcher.Celo.EpochBlockOperations.ValidatorAndGroupPayments do + @moduledoc """ + Fetches validator and group payments for the epoch block. + """ + import Ecto.Query, only: [from: 2] + + alias Explorer.Chain.Cache.CeloCoreContracts + alias Explorer.Chain.Log + alias Explorer.Repo + alias Indexer.Transform.Celo.ValidatorEpochPaymentDistributions + + @spec fetch(%{ + :block_hash => EthereumJSONRPC.hash(), + :block_number => EthereumJSONRPC.block_number() + }) :: {:ok, list()} + def fetch(%{block_number: block_number, block_hash: block_hash} = _pending_operation) do + epoch_payment_distributions_signature = ValidatorEpochPaymentDistributions.signature() + {:ok, validators_contract_address} = CeloCoreContracts.get_address(:validators, block_number) + + query = + from( + log in Log, + where: + log.block_hash == ^block_hash and + log.address_hash == ^validators_contract_address and + log.first_topic == ^epoch_payment_distributions_signature and + is_nil(log.transaction_hash), + select: log + ) + + payments = + query + |> Repo.all() + |> ValidatorEpochPaymentDistributions.parse() + |> process_distribution_events(block_hash) + + {:ok, payments} + end + + defp process_distribution_events(distribution_events, block_hash) do + distribution_events + |> Enum.map(fn %{ + validator_address: validator_address, + validator_payment: validator_payment, + group_address: group_address, + group_payment: group_payment + } -> + [ + %{ + block_hash: block_hash, + account_address_hash: validator_address, + amount: validator_payment, + associated_account_address_hash: group_address, + type: :validator + }, + %{ + block_hash: block_hash, + account_address_hash: group_address, + amount: group_payment, + associated_account_address_hash: validator_address, + type: :group + } + ] + end) + |> Enum.concat() + end +end diff --git a/apps/indexer/lib/indexer/fetcher/celo/epoch_block_operations/voter_payments.ex b/apps/indexer/lib/indexer/fetcher/celo/epoch_block_operations/voter_payments.ex new file mode 100644 index 0000000000..d526feb0c7 --- /dev/null +++ b/apps/indexer/lib/indexer/fetcher/celo/epoch_block_operations/voter_payments.ex @@ -0,0 +1,201 @@ +defmodule Indexer.Fetcher.Celo.EpochBlockOperations.VoterPayments do + @moduledoc """ + Fetches voter payments for the epoch block. + """ + import Ecto.Query, only: [from: 2] + + import Explorer.Helper, only: [decode_data: 2] + import Indexer.Fetcher.Celo.Helper, only: [abi_to_method_id: 1] + import Indexer.Helper, only: [read_contracts_with_retries: 4] + + alias Explorer.Repo + alias Indexer.Fetcher.Celo.ValidatorGroupVotes + + alias Explorer.Chain.{ + Cache.CeloCoreContracts, + Celo.ValidatorGroupVote, + Hash, + Log + } + + require Logger + + @repeated_request_max_retries 3 + + @epoch_rewards_distributed_to_voters_topic "0x91ba34d62474c14d6c623cd322f4256666c7a45b7fdaa3378e009d39dfcec2a7" + + @get_active_votes_for_group_by_account_abi [ + %{ + "name" => "getActiveVotesForGroupByAccount", + "type" => "function", + "payable" => false, + "constant" => true, + "stateMutability" => "view", + "inputs" => [ + %{"name" => "group", "type" => "address"}, + %{"name" => "account", "type" => "address"} + ], + "outputs" => [ + %{"type" => "uint256"} + ] + } + ] + + @get_active_votes_for_group_by_account_method_id @get_active_votes_for_group_by_account_abi + |> abi_to_method_id() + + @spec fetch( + %{ + :block_hash => EthereumJSONRPC.hash(), + :block_number => EthereumJSONRPC.block_number() + }, + EthereumJSONRPC.json_rpc_named_arguments() + ) :: {:error, list()} | {:ok, list()} + def fetch( + %{block_number: block_number, block_hash: block_hash} = pending_operation, + json_rpc_named_arguments + ) do + :ok = ValidatorGroupVotes.fetch(block_number) + + {:ok, election_contract_address} = CeloCoreContracts.get_address(:election, block_number) + + elected_groups_query = + from( + l in Log, + where: + l.block_hash == ^block_hash and + l.address_hash == ^election_contract_address and + l.first_topic == ^@epoch_rewards_distributed_to_voters_topic and + is_nil(l.transaction_hash), + select: fragment("SUBSTRING(? from 13)", l.second_topic) + ) + + query = + from( + v in ValidatorGroupVote, + where: + v.group_address_hash in subquery(elected_groups_query) and + v.block_number <= ^block_number, + distinct: true, + select: {v.account_address_hash, v.group_address_hash} + ) + + accounts_with_activated_votes = + query + |> Repo.all() + |> Enum.map(fn + {account_address_hash, group_address_hash} -> + { + Hash.to_string(account_address_hash), + Hash.to_string(group_address_hash) + } + end) + + requests = + accounts_with_activated_votes + |> Enum.map(fn {account_address_hash, group_address_hash} -> + (block_number - 1)..block_number + |> Enum.map(fn block_number -> + %{ + contract_address: election_contract_address, + method_id: @get_active_votes_for_group_by_account_method_id, + args: [ + group_address_hash, + account_address_hash + ], + block_number: block_number + } + end) + end) + |> Enum.concat() + + {responses, []} = + read_contracts_with_retries( + requests, + @get_active_votes_for_group_by_account_abi, + json_rpc_named_arguments, + @repeated_request_max_retries + ) + + diffs = + responses + |> Enum.chunk_every(2) + |> Enum.map(fn + [ok: [votes_before], ok: [votes_after]] + when is_integer(votes_before) and + is_integer(votes_after) -> + votes_after - votes_before + end) + + # WARN: we do not count Revoked/Activated votes for the last epoch, but + # should we? + # + # See https://github.com/fedor-ivn/celo-blockscout/tree/master/apps/indexer/lib/indexer/fetcher/celo_epoch_data.ex#L179-L187 + # There is no case when those events occur in the epoch block. + rewards = + accounts_with_activated_votes + |> Enum.zip_with( + diffs, + fn {account_address_hash, group_address_hash}, diff -> + %{ + block_hash: block_hash, + account_address_hash: account_address_hash, + amount: diff, + associated_account_address_hash: group_address_hash, + type: :voter + } + end + ) + + ok_or_error = validate_voter_rewards(pending_operation, rewards) + + {ok_or_error, rewards} + end + + # Validates voter rewards by comparing the sum of what we got from the + # `EpochRewardsDistributedToVoters` event and the sum of what we calculated + # manually by fetching the votes for each account that has or had an activated + # vote. + defp validate_voter_rewards(pending_operation, voter_rewards) do + manual_voters_total = voter_rewards |> Enum.map(& &1.amount) |> Enum.sum() + {:ok, election_contract_address} = CeloCoreContracts.get_address(:election, pending_operation.block_number) + + query = + from( + l in Log, + where: + l.block_hash == ^pending_operation.block_hash and + l.address_hash == ^election_contract_address and + l.first_topic == ^@epoch_rewards_distributed_to_voters_topic and + is_nil(l.transaction_hash), + select: l.data + ) + + voter_rewards_from_event_total = + query + |> Repo.all() + |> Enum.map(fn data -> + [amount] = decode_data(data, [{:uint, 256}]) + amount + end) + |> Enum.sum() + + voter_rewards_count = Enum.count(voter_rewards) + voter_rewards_diff = voter_rewards_from_event_total - manual_voters_total + + if voter_rewards_diff < voter_rewards_count or voter_rewards_count == 0 do + :ok + else + Logger.warning(fn -> + [ + "Total voter rewards do not match. ", + "Amount calculated manually: #{manual_voters_total}. ", + "Amount got from `EpochRewardsDistributedToVoters` events: #{voter_rewards_from_event_total}. ", + "Voter rewards count: #{voter_rewards_count}." + ] + end) + + :error + end + end +end diff --git a/apps/indexer/lib/indexer/fetcher/celo/epoch_logs.ex b/apps/indexer/lib/indexer/fetcher/celo/epoch_logs.ex new file mode 100644 index 0000000000..4d12ab9c8a --- /dev/null +++ b/apps/indexer/lib/indexer/fetcher/celo/epoch_logs.ex @@ -0,0 +1,121 @@ +defmodule Indexer.Fetcher.Celo.EpochLogs do + @moduledoc """ + Fetches logs that are not associated which are not linked to transaction, but + to the block. + """ + + import Explorer.Chain.Celo.Helper, only: [epoch_block_number?: 1] + + alias EthereumJSONRPC.Logs + alias Explorer.Chain.Cache.CeloCoreContracts + alias Indexer.Helper, as: IndexerHelper + + alias Explorer.Chain.TokenTransfer + alias Indexer.Transform.Celo.ValidatorEpochPaymentDistributions + + require Logger + + @max_request_retries 3 + + @epoch_block_targets [ + # TargetVotingYieldUpdated + epoch_rewards: "0x49d8cdfe05bae61517c234f65f4088454013bafe561115126a8fe0074dc7700e", + celo_token: TokenTransfer.constant(), + usd_token: TokenTransfer.constant(), + validators: ValidatorEpochPaymentDistributions.signature(), + # ValidatorScoreUpdated + validators: "0xedf9f87e50e10c533bf3ae7f5a7894ae66c23e6cbbe8773d7765d20ad6f995e9", + # EpochRewardsDistributedToVoters + election: "0x91ba34d62474c14d6c623cd322f4256666c7a45b7fdaa3378e009d39dfcec2a7" + ] + + @default_block_targets [ + # GasPriceMinimumUpdated + gas_price_minimum: "0x6e53b2f8b69496c2a175588ad1326dbabe2f66df4d82f817aeca52e3474807fb" + ] + + @spec fetch( + [Indexer.Transform.Blocks.block()], + EthereumJSONRPC.json_rpc_named_arguments() + ) :: Logs.t() + def fetch(blocks, json_rpc_named_arguments) + + def fetch(blocks, json_rpc_named_arguments) do + if Application.get_env(:explorer, :chain_type) == :celo do + do_fetch(blocks, json_rpc_named_arguments) + else + [] + end + end + + defp do_fetch(blocks, json_rpc_named_arguments) do + requests = + blocks + |> Enum.reduce({[], 0}, &blocks_reducer/2) + |> elem(0) + |> Enum.reverse() + |> Enum.concat() + + with {:ok, responses} <- do_requests(requests, json_rpc_named_arguments), + {:ok, logs} <- Logs.from_responses(responses) do + logs + |> Enum.filter(&(&1.transaction_hash == &1.block_hash)) + |> Enum.map(&Map.put(&1, :transaction_hash, nil)) + end + end + + # Workaround in order to fix block fetcher tests. + # + # If the requests is empty, we still send the requests to the JSON RPC + defp do_requests(requests, json_rpc_named_arguments) do + if Enum.empty?(requests) do + {:ok, []} + else + IndexerHelper.repeated_batch_rpc_call( + requests, + json_rpc_named_arguments, + fn message -> "Could not fetch epoch logs: #{message}" end, + @max_request_retries + ) + end + end + + defp blocks_reducer(%{number: number}, {acc, start_request_id}) do + targets = + @default_block_targets ++ + if epoch_block_number?(number) do + @epoch_block_targets + else + [] + end + + requests = + targets + |> Enum.map(fn {contract_atom, topic} -> + res = CeloCoreContracts.get_address(contract_atom, number) + {res, topic} + end) + |> Enum.split_with(&match?({{:ok, _address}, _topic}, &1)) + |> tap(fn {_, not_found} -> + if not Enum.empty?(not_found) do + Logger.info("Could not fetch addresses for the following contract atoms: #{inspect(not_found)}") + end + end) + |> elem(0) + |> Enum.with_index(start_request_id) + |> Enum.map(fn {{{:ok, address}, topic}, request_id} -> + Logs.request( + request_id, + %{ + from_block: number, + to_block: number, + address: address, + topics: [topic] + } + ) + end) + + next_start_request_id = start_request_id + length(requests) + {[requests | acc], next_start_request_id} + end +end diff --git a/apps/indexer/lib/indexer/fetcher/celo/helper.ex b/apps/indexer/lib/indexer/fetcher/celo/helper.ex new file mode 100644 index 0000000000..ec178397cd --- /dev/null +++ b/apps/indexer/lib/indexer/fetcher/celo/helper.ex @@ -0,0 +1,29 @@ +defmodule Indexer.Fetcher.Celo.Helper do + @moduledoc """ + Helper functions for the Celo fetchers. + """ + + @doc """ + Extracts the method ID from an ABI specification. + + ## Parameters + - `method` ([map()] | map()): The ABI specification, either as a single map + or a list containing one map. + + ## Returns + - `binary()`: The method ID extracted from the ABI specification. + + ## Examples + + iex> Indexer.Fetcher.Celo.Helper.abi_to_method_id([%{"name" => "transfer", "type" => "function", "inputs" => [%{"name" => "to", "type" => "address"}]}]) + <<26, 105, 82, 48>> + + """ + @spec abi_to_method_id([map()] | map()) :: binary() + def abi_to_method_id([method]), do: abi_to_method_id(method) + + def abi_to_method_id(method) when is_map(method) do + [parsed_method] = ABI.parse_specification([method]) + parsed_method.method_id + end +end diff --git a/apps/indexer/lib/indexer/fetcher/celo/validator_group_votes.ex b/apps/indexer/lib/indexer/fetcher/celo/validator_group_votes.ex new file mode 100644 index 0000000000..f51e736e00 --- /dev/null +++ b/apps/indexer/lib/indexer/fetcher/celo/validator_group_votes.ex @@ -0,0 +1,217 @@ +defmodule Indexer.Fetcher.Celo.ValidatorGroupVotes do + @moduledoc """ + Fetches validator group votes from the Celo blockchain. + """ + + use GenServer + use Indexer.Fetcher + + import Explorer.Helper, + only: [ + truncate_address_hash: 1, + safe_parse_non_negative_integer: 1 + ] + + alias EthereumJSONRPC.Logs + alias Explorer.Application.Constants + alias Explorer.Chain + alias Explorer.Chain.Cache.CeloCoreContracts + alias Indexer.Helper, as: IndexerHelper + alias Indexer.Transform.Addresses + + require Logger + + @last_fetched_block_key "celo_validator_group_votes_last_fetched_block_number" + + @max_request_retries 3 + + @validator_group_vote_activated_topic "0x45aac85f38083b18efe2d441a65b9c1ae177c78307cb5a5d4aec8f7dbcaeabfe" + @validator_group_active_vote_revoked_topic "0xae7458f8697a680da6be36406ea0b8f40164915ac9cc40c0dad05a2ff6e8c6a8" + + @spec fetch(block_number :: EthereumJSONRPC.block_number()) :: :ok + def fetch(block_number) do + GenServer.call(__MODULE__, {:fetch, block_number}) + end + + def child_spec(start_link_arguments) do + spec = %{ + id: __MODULE__, + start: {__MODULE__, :start_link, start_link_arguments}, + type: :worker + } + + Supervisor.child_spec(spec, []) + end + + def start_link(args, gen_server_options \\ []) do + GenServer.start_link(__MODULE__, args, Keyword.put_new(gen_server_options, :name, __MODULE__)) + end + + @impl GenServer + def init(args) do + Logger.metadata(fetcher: :celo_validator_group_votes) + + { + :ok, + %{ + config: %{ + batch_size: Application.get_env(:indexer, __MODULE__)[:batch_size], + json_rpc_named_arguments: args[:json_rpc_named_arguments] + }, + data: %{} + }, + {:continue, :ok} + } + end + + @impl GenServer + def handle_continue( + :ok, + %{ + config: %{ + batch_size: batch_size, + json_rpc_named_arguments: json_rpc_named_arguments + } + } = state + ) do + {:ok, latest_block_number} = + EthereumJSONRPC.fetch_block_number_by_tag( + "latest", + json_rpc_named_arguments + ) + + Logger.info("Fetching votes up to latest block number #{latest_block_number}") + + fetch_up_to_block_number(latest_block_number, batch_size, json_rpc_named_arguments) + + {:noreply, state} + end + + @impl GenServer + def handle_info({ref, _result}, state) do + Process.demonitor(ref, [:flush]) + {:noreply, state} + end + + @impl GenServer + def handle_call( + {:fetch, block_number}, + _from, + %{ + config: %{ + batch_size: batch_size, + json_rpc_named_arguments: json_rpc_named_arguments + } + } = state + ) do + Logger.info("Fetching votes on demand up to block number #{block_number}") + + fetch_up_to_block_number(block_number, batch_size, json_rpc_named_arguments) + + {:reply, :ok, state} + end + + defp fetch_up_to_block_number(block_number, batch_size, json_rpc_named_arguments) do + {:ok, last_fetched_block_number} = + @last_fetched_block_key + |> Constants.get_constant_value() + |> case do + nil -> CeloCoreContracts.get_first_update_block_number(:election) + value -> safe_parse_non_negative_integer(value) + end + + if last_fetched_block_number < block_number do + block_range = last_fetched_block_number..block_number + + block_range + |> IndexerHelper.range_chunk_every(batch_size) + |> Enum.each(&process_chunk(&1, block_range, json_rpc_named_arguments)) + + Logger.info("Fetched validator group votes up to block number #{block_number}") + else + Logger.info("No new validator group votes to fetch") + end + end + + defp process_chunk(_..chunk_to_block = chunk, block_range, json_rpc_named_arguments) do + validator_group_votes = + chunk + |> fetch_logs_chunk(block_range, json_rpc_named_arguments) + |> Enum.map(&log_to_entry/1) + + addresses_params = + Addresses.extract_addresses(%{ + celo_validator_group_votes: validator_group_votes + }) + + {:ok, _imported} = + Chain.import(%{ + addresses: %{params: addresses_params}, + celo_validator_group_votes: %{params: validator_group_votes} + }) + + Constants.set_constant_value(@last_fetched_block_key, to_string(chunk_to_block)) + + :ok + end + + defp fetch_logs_chunk( + chunk_from_block..chunk_to_block, + from_block..to_block, + json_rpc_named_arguments + ) do + IndexerHelper.log_blocks_chunk_handling(chunk_from_block, chunk_to_block, from_block, to_block, nil, :L1) + + {:ok, election_contract_address} = CeloCoreContracts.get_address(:election, chunk_from_block) + + requests = + [ + @validator_group_active_vote_revoked_topic, + @validator_group_vote_activated_topic + ] + |> Enum.with_index() + |> Enum.map(fn {topic, request_id} -> + Logs.request( + request_id, + %{ + from_block: chunk_from_block, + to_block: chunk_to_block, + address: election_contract_address, + topics: [topic] + } + ) + end) + + {:ok, responses} = + IndexerHelper.repeated_batch_rpc_call( + requests, + json_rpc_named_arguments, + fn message -> Logger.error("Could not fetch logs: #{message}") end, + @max_request_retries + ) + + {:ok, logs} = Logs.from_responses(responses) + + logs + end + + defp log_to_entry(log) do + type = + case log.first_topic do + @validator_group_vote_activated_topic -> :activated + @validator_group_active_vote_revoked_topic -> :revoked + end + + account_address_hash = truncate_address_hash(log.second_topic) + group_address_hash = truncate_address_hash(log.third_topic) + + %{ + account_address_hash: account_address_hash, + group_address_hash: group_address_hash, + type: type, + block_number: log.block_number, + block_hash: log.block_hash, + transaction_hash: log.transaction_hash + } + end +end diff --git a/apps/indexer/lib/indexer/fetcher/internal_transaction.ex b/apps/indexer/lib/indexer/fetcher/internal_transaction.ex index f847af2ebc..91d23414d9 100644 --- a/apps/indexer/lib/indexer/fetcher/internal_transaction.ex +++ b/apps/indexer/lib/indexer/fetcher/internal_transaction.ex @@ -10,15 +10,21 @@ defmodule Indexer.Fetcher.InternalTransaction do require Logger - import Indexer.Block.Fetcher, only: [async_import_coin_balances: 2] + import Indexer.Block.Fetcher, + only: [ + async_import_coin_balances: 2, + async_import_token_balances: 2, + token_transfers_merge_token: 2 + ] alias EthereumJSONRPC.Utility.RangesHelper alias Explorer.Chain - alias Explorer.Chain.Block + alias Explorer.Chain.{Block, Hash} alias Explorer.Chain.Cache.{Accounts, Blocks} alias Indexer.{BufferedTask, Tracer} alias Indexer.Fetcher.InternalTransaction.Supervisor, as: InternalTransactionSupervisor - alias Indexer.Transform.Addresses + alias Indexer.Transform.Celo.TransactionTokenTransfers, as: CeloTransactionTokenTransfers + alias Indexer.Transform.{Addresses, AddressTokenBalances} @behaviour BufferedTask @@ -280,8 +286,29 @@ defmodule Indexer.Fetcher.InternalTransaction do internal_transactions_and_empty_block_numbers = internal_transactions_params_marked ++ empty_block_numbers + celo_token_transfers_params = + %{token_transfers: celo_token_transfers, tokens: celo_tokens} = + if Application.get_env(:explorer, :chain_type) == :celo do + block_number_to_block_hash = + unique_numbers + |> Chain.block_hash_by_number() + |> Map.new(fn + {block_number, block_hash} -> + {block_number, Hash.to_string(block_hash)} + end) + + CeloTransactionTokenTransfers.parse_internal_transactions( + internal_transactions_params_marked, + block_number_to_block_hash + ) + else + %{token_transfers: [], tokens: []} + end + imports = Chain.import(%{ + token_transfers: %{params: celo_token_transfers}, + tokens: %{params: celo_tokens}, addresses: %{params: addresses_params}, internal_transactions: %{params: internal_transactions_and_empty_block_numbers, with: :blockless_changeset}, timeout: :infinity @@ -296,6 +323,8 @@ defmodule Indexer.Fetcher.InternalTransaction do address_hash_to_fetched_balance_block_number: address_hash_to_block_number }) + async_import_celo_token_balances(celo_token_transfers_params) + {:error, step, reason, _changes_so_far} -> Logger.error( fn -> @@ -411,4 +440,28 @@ defmodule Indexer.Fetcher.InternalTransaction do metadata: [fetcher: :internal_transaction] ] end + + defp async_import_celo_token_balances(%{token_transfers: token_transfers, tokens: tokens}) do + if Application.get_env(:explorer, :chain_type) == :celo do + token_transfers_with_token = token_transfers_merge_token(token_transfers, tokens) + + address_token_balances = + %{token_transfers_params: token_transfers_with_token} + |> AddressTokenBalances.params_set() + |> Enum.map(fn %{address_hash: address_hash, token_contract_address_hash: token_contract_address_hash} = entry -> + with {:ok, address_hash} <- Hash.Address.cast(address_hash), + {:ok, token_contract_address_hash} <- Hash.Address.cast(token_contract_address_hash) do + entry + |> Map.put(:address_hash, address_hash) + |> Map.put(:token_contract_address_hash, token_contract_address_hash) + else + error -> Logger.error("Failed to cast string to hash: #{inspect(error)}") + end + end) + + async_import_token_balances(%{address_token_balances: address_token_balances}, false) + else + :ok + end + end end diff --git a/apps/indexer/lib/indexer/helper.ex b/apps/indexer/lib/indexer/helper.ex index 23043f7802..b5a143e76b 100644 --- a/apps/indexer/lib/indexer/helper.ex +++ b/apps/indexer/lib/indexer/helper.ex @@ -200,6 +200,36 @@ defmodule Indexer.Helper do ] end + @doc """ + Splits a given range into chunks of the specified size. + + ## Parameters + - `range`: The range to be split into chunks. + - `chunk_size`: The size of each chunk. + + ## Returns + - A stream of ranges, each representing a chunk of the specified size. + + ## Examples + + iex> Indexer.Helper.range_chunk_every(1..10, 3) + #Stream<...> + + iex> Enum.to_list(Indexer.Helper.range_chunk_every(1..10, 3)) + [1..3, 4..6, 7..9, 10..10] + """ + @spec range_chunk_every(Range.t(), non_neg_integer()) :: Enum.t() + def range_chunk_every(from..to, chunk_size) do + chunks_number = floor((to - from + 1) / chunk_size) + + 0..chunks_number + |> Stream.map(fn current_chunk -> + chunk_start = from + chunk_size * current_chunk + chunk_end = min(chunk_start + chunk_size - 1, to) + chunk_start..chunk_end + end) + end + @doc """ Retrieves event logs from Ethereum-like blockchains within a specified block range for a given address and set of topics using JSON-RPC. @@ -314,26 +344,35 @@ defmodule Indexer.Helper do end @doc """ - Retrieves decoded results of `eth_call` requests to contracts, with retry logic for handling errors. + Retrieves decoded results of `eth_call` requests to contracts, with retry + logic for handling errors. - The function attempts the specified number of retries, with a progressive delay between - each retry, for each `eth_call` request. If, after all retries, some requests remain - unsuccessful, it returns a list of unique error messages encountered. + The function attempts the specified number of retries, with a progressive + delay between each retry, for each `eth_call` request. If, after all + retries, some requests remain unsuccessful, it returns a list of unique + error messages encountered. ## Parameters - - `requests`: A list of `EthereumJSONRPC.Contract.call()` instances describing the parameters - for `eth_call`, including the contract address and method selector. - - `abi`: A list of maps providing the ABI that describes the input parameters and output - format for the contract functions. - - `json_rpc_named_arguments`: Configuration parameters for the JSON RPC connection. - - `retries_left`: The number of retries allowed for any `eth_call` that returns an error. + - `requests`: A list of `EthereumJSONRPC.Contract.call()` instances + describing the parameters for `eth_call`, including the + contract address and method selector. + - `abi`: A list of maps providing the ABI that describes the input + parameters and output format for the contract functions. + - `json_rpc_named_arguments`: Configuration parameters for the JSON RPC + connection. + - `retries_left`: The number of retries allowed for any `eth_call` that + returns an error. + - `log_error?` (optional): A boolean indicating whether to log error + messages on retries. Defaults to `true`. ## Returns - `{responses, errors}` where: - - `responses`: A list of tuples `{status, result}`, where `result` is the decoded response - from the corresponding `eth_call` if `status` is `:ok`, or the error message - if `status` is `:error`. - - `errors`: A list of error messages, if any element in `responses` contains `:error`. + - `responses`: A list of tuples `{status, result}`, where `result` is the + decoded response from the corresponding `eth_call` if + `status` is `:ok`, or the error message if `status` is + `:error`. + - `errors`: A list of error messages, if any element in `responses` + contains `:error`. """ @spec read_contracts_with_retries( [EthereumJSONRPC.Contract.call()], @@ -341,12 +380,12 @@ defmodule Indexer.Helper do EthereumJSONRPC.json_rpc_named_arguments(), integer() ) :: {[{:ok | :error, any()}], list()} - def read_contracts_with_retries(requests, abi, json_rpc_named_arguments, retries_left) + def read_contracts_with_retries(requests, abi, json_rpc_named_arguments, retries_left, log_error? \\ true) when is_list(requests) and is_list(abi) and is_integer(retries_left) do - do_read_contracts_with_retries(requests, abi, json_rpc_named_arguments, retries_left, 0) + do_read_contracts_with_retries(requests, abi, json_rpc_named_arguments, retries_left, 0, log_error?) end - defp do_read_contracts_with_retries(requests, abi, json_rpc_named_arguments, retries_left, retries_done) do + defp do_read_contracts_with_retries(requests, abi, json_rpc_named_arguments, retries_left, retries_done, log_error?) do responses = ContractReader.query_contracts(requests, abi, json_rpc_named_arguments: json_rpc_named_arguments) error_messages = @@ -359,18 +398,28 @@ defmodule Indexer.Helper do end end) - if error_messages == [] do - {responses, []} - else - retries_left = retries_left - 1 + retries_left = retries_left - 1 - if retries_left <= 0 do + cond do + error_messages == [] -> + {responses, []} + + retries_left <= 0 -> + if log_error?, do: Logger.error("#{List.first(error_messages)}.") {responses, Enum.uniq(error_messages)} - else - Logger.error("#{List.first(error_messages)}. Retrying...") + + true -> + if log_error?, do: Logger.error("#{List.first(error_messages)}. Retrying...") pause_before_retry(retries_done) - do_read_contracts_with_retries(requests, abi, json_rpc_named_arguments, retries_left, retries_done + 1) - end + + do_read_contracts_with_retries( + requests, + abi, + json_rpc_named_arguments, + retries_left, + retries_done + 1, + log_error? + ) end end @@ -387,7 +436,7 @@ defmodule Indexer.Helper do - `json_rpc_named_arguments`: Configuration parameters for the JSON RPC connection. - `error_message_generator`: A function that generates a string containing the error message returned by the RPC call. - - `retries_left`: The number of retries allowed for any RPC call that returns an error. + - `max_retries`: The number of retries allowed for any RPC call that returns an error. ## Returns - `{:ok, responses}`: When all calls are successful, `responses` is a list of standard @@ -399,9 +448,9 @@ defmodule Indexer.Helper do """ @spec repeated_batch_rpc_call([Transport.request()], EthereumJSONRPC.json_rpc_named_arguments(), fun(), integer()) :: {:error, any()} | {:ok, any()} - def repeated_batch_rpc_call(requests, json_rpc_named_arguments, error_message_generator, retries_left) - when is_list(requests) and is_function(error_message_generator) and is_integer(retries_left) do - do_repeated_batch_rpc_call(requests, json_rpc_named_arguments, error_message_generator, retries_left, 0) + def repeated_batch_rpc_call(requests, json_rpc_named_arguments, error_message_generator, max_retries) + when is_list(requests) and is_function(error_message_generator) and is_integer(max_retries) do + do_repeated_batch_rpc_call(requests, json_rpc_named_arguments, error_message_generator, max_retries, 0) end # credo:disable-for-next-line Credo.Check.Refactor.CyclomaticComplexity diff --git a/apps/indexer/lib/indexer/supervisor.ex b/apps/indexer/lib/indexer/supervisor.ex index ecb32417fb..641b4ac654 100644 --- a/apps/indexer/lib/indexer/supervisor.ex +++ b/apps/indexer/lib/indexer/supervisor.ex @@ -189,6 +189,12 @@ defmodule Indexer.Supervisor do configure(ArbitrumRollupMessagesCatchup.Supervisor, [ [json_rpc_named_arguments: json_rpc_named_arguments, memory_monitor: memory_monitor] ]), + configure(Indexer.Fetcher.Celo.ValidatorGroupVotes.Supervisor, [ + [json_rpc_named_arguments: json_rpc_named_arguments, memory_monitor: memory_monitor] + ]), + configure(Indexer.Fetcher.Celo.EpochBlockOperations.Supervisor, [ + [json_rpc_named_arguments: json_rpc_named_arguments, memory_monitor: memory_monitor] + ]), {Indexer.Fetcher.Beacon.Blob.Supervisor, [[memory_monitor: memory_monitor]]}, # Out-of-band fetchers @@ -282,6 +288,7 @@ defmodule Indexer.Supervisor do end defp configure(process, opts) do + # todo: shouldn't we pay attention to process.disabled?() predicate? if Application.get_env(:indexer, process)[:enabled] do [{process, opts}] else diff --git a/apps/indexer/lib/indexer/transform/address_coin_balances.ex b/apps/indexer/lib/indexer/transform/address_coin_balances.ex index 9f1935da46..e85cd1ffa1 100644 --- a/apps/indexer/lib/indexer/transform/address_coin_balances.ex +++ b/apps/indexer/lib/indexer/transform/address_coin_balances.ex @@ -90,15 +90,39 @@ defmodule Indexer.Transform.AddressCoinBalances do ) when is_integer(block_number) and is_binary(from_address_hash) do # a transaction MUST have a `from_address_hash` - acc = MapSet.put(initial, %{address_hash: from_address_hash, block_number: block_number}) - - # `to_address_hash` is optional - case transaction_params do - %{to_address_hash: to_address_hash} when is_binary(to_address_hash) -> - MapSet.put(acc, %{address_hash: to_address_hash, block_number: block_number}) + initial + |> MapSet.put(%{address_hash: from_address_hash, block_number: block_number}) + |> (&(case transaction_params do + %{to_address_hash: to_address_hash} when is_binary(to_address_hash) -> + MapSet.put(&1, %{address_hash: to_address_hash, block_number: block_number}) + + _ -> + &1 + end)).() + |> (&transactions_params_chain_type_fields_reducer(transaction_params, &1)).() + end - _ -> - acc + if Application.compile_env(:explorer, :chain_type) == :celo do + import Explorer.Chain.SmartContract, only: [burn_address_hash_string: 0] + + @burn_address_hash_string burn_address_hash_string() + + # todo: subject for deprecation, since celo transactions with + # gatewayFeeRecipient are deprecated + defp transactions_params_chain_type_fields_reducer( + %{ + block_number: block_number, + gas_fee_recipient_address_hash: recipient_address_hash, + gas_token_contract_address_hash: nil + }, + initial + ) + when is_integer(block_number) and + is_binary(recipient_address_hash) and + recipient_address_hash != @burn_address_hash_string do + MapSet.put(initial, %{address_hash: recipient_address_hash, block_number: block_number}) end end + + defp transactions_params_chain_type_fields_reducer(_, acc), do: acc end diff --git a/apps/indexer/lib/indexer/transform/addresses.ex b/apps/indexer/lib/indexer/transform/addresses.ex index af645ba344..73c48375f8 100644 --- a/apps/indexer/lib/indexer/transform/addresses.ex +++ b/apps/indexer/lib/indexer/transform/addresses.ex @@ -155,6 +155,19 @@ defmodule Indexer.Transform.Addresses do [ %{from: :l2_token_address, to: :hash} ] + ], + celo_election_rewards: [ + [ + %{from: :account_address_hash, to: :hash} + ] + ], + celo_validator_group_votes: [ + [ + %{from: :account_address_hash, to: :hash} + ], + [ + %{from: :group_address_hash, to: :hash} + ] ] } @@ -176,7 +189,7 @@ defmodule Indexer.Transform.Addresses do Blocks have their `miner_hash` extracted. - iex> Indexer.Addresses.extract_addresses( + iex> Indexer.Transform.Addresses.extract_addresses( ...> %{ ...> blocks: [ ...> %{ @@ -196,7 +209,7 @@ defmodule Indexer.Transform.Addresses do Internal transactions can have their `from_address_hash`, `to_address_hash` and/or `created_contract_address_hash` extracted. - iex> Indexer.Addresses.extract_addresses( + iex> Indexer.Transform.Addresses.extract_addresses( ...> %{ ...> internal_transactions: [ ...> %{ @@ -233,7 +246,7 @@ defmodule Indexer.Transform.Addresses do Transactions can have their `from_address_hash` and/or `to_address_hash` extracted. - iex> Indexer.Addresses.extract_addresses( + iex> Indexer.Transform.Addresses.extract_addresses( ...> %{ ...> transactions: [ ...> %{ @@ -269,7 +282,7 @@ defmodule Indexer.Transform.Addresses do Logs can have their `address_hash` extracted. - iex> Indexer.Addresses.extract_addresses( + iex> Indexer.Transform.Addresses.extract_addresses( ...> %{ ...> logs: [ ...> %{ @@ -288,7 +301,7 @@ defmodule Indexer.Transform.Addresses do When the same address is mentioned multiple times, the greatest `block_number` is used - iex> Indexer.Addresses.extract_addresses( + iex> Indexer.Transform.Addresses.extract_addresses( ...> %{ ...> blocks: [ ...> %{ @@ -341,7 +354,7 @@ defmodule Indexer.Transform.Addresses do When a contract is created and then used in internal transactions and transaction in the same fetched data, the `created_contract_code` is merged with the greatest `block_number` - iex> Indexer.Addresses.extract_addresses( + iex> Indexer.Transform.Addresses.extract_addresses( ...> %{ ...> internal_transactions: [ ...> %{ @@ -467,6 +480,17 @@ defmodule Indexer.Transform.Addresses do %{ optional(:l2_token_address) => String.t() } + ], + optional(:celo_election_rewards) => [ + %{ + required(:account_address_hash) => String.t() + } + ], + optional(:celo_validator_group_votes) => [ + %{ + required(:account_address_hash) => String.t(), + required(:group_address_hash) => String.t() + } ] }) :: [params] def extract_addresses(fetched_data, options \\ []) when is_map(fetched_data) and is_list(options) do diff --git a/apps/indexer/lib/indexer/transform/celo/transaction_gas_tokens.ex b/apps/indexer/lib/indexer/transform/celo/transaction_gas_tokens.ex new file mode 100644 index 0000000000..7857a93215 --- /dev/null +++ b/apps/indexer/lib/indexer/transform/celo/transaction_gas_tokens.ex @@ -0,0 +1,42 @@ +defmodule Indexer.Transform.Celo.TransactionGasTokens do + @moduledoc """ + Helper functions for extracting tokens specified as gas fee currency. + """ + + alias Explorer.Chain.Hash + + @doc """ + Parses transactions and extracts tokens specified as gas fee currency. + """ + @spec parse([ + %{ + optional(:gas_token_contract_address_hash) => Hash.Address.t() | nil + } + ]) :: [ + %{ + contract_address_hash: String.t(), + type: String.t() + } + ] + def parse(transactions) do + if Application.get_env(:explorer, :chain_type) == :celo do + transactions + |> Enum.reduce( + MapSet.new(), + fn + %{gas_token_contract_address_hash: address_hash}, acc when not is_nil(address_hash) -> + MapSet.put(acc, %{ + contract_address_hash: address_hash, + type: "ERC-20" + }) + + _, acc -> + acc + end + ) + |> MapSet.to_list() + else + [] + end + end +end diff --git a/apps/indexer/lib/indexer/transform/celo/transaction_token_transfers.ex b/apps/indexer/lib/indexer/transform/celo/transaction_token_transfers.ex new file mode 100644 index 0000000000..2efa6b74de --- /dev/null +++ b/apps/indexer/lib/indexer/transform/celo/transaction_token_transfers.ex @@ -0,0 +1,138 @@ +defmodule Indexer.Transform.Celo.TransactionTokenTransfers do + @moduledoc """ + Helper functions for generating ERC20 token transfers from native Celo coin + transfers. + + CELO has a feature referred to as "token duality", where the native chain + asset (CELO) can be used as both a native chain currency and as an ERC-20 + token. Unfortunately native chain asset transfers do not emit ERC-20 transfer + events, which requires the artificial creation of entries in the + `token_transfers` table. + """ + require Logger + + import Indexer.Transform.TokenTransfers, + only: [ + filter_tokens_for_supply_update: 1 + ] + + alias Explorer.Chain.Cache.CeloCoreContracts + alias Explorer.Chain.Hash + alias Indexer.Fetcher.TokenTotalSupplyUpdater + @token_type "ERC-20" + @transaction_buffer_size 20_000 + + @doc """ + In order to avoid conflicts with real token transfers, for native token + transfers we put a negative `log_index`. + + Each transaction within the block is assigned a so-called _buffer_ of + #{@transaction_buffer_size} entries. Thus, according to the formula, + transactions with indices 0, 1, 2 would have log indices -20000, -40000, + -60000. + + The spare intervals between the log indices (0..-19_999, -20_001..-39_999, + -40_001..59_999) are reserved for native token transfers fetched from + internal transactions. + """ + @spec parse_transactions([ + %{ + required(:value) => non_neg_integer(), + optional(:to_address_hash) => Hash.Address.t() | nil, + optional(:created_contract_address_hash) => Hash.Address.t() | nil + } + ]) :: %{ + token_transfers: list(), + tokens: list() + } + def parse_transactions(transactions) do + token_transfers = + if Application.get_env(:explorer, :chain_type) == :celo do + transactions + |> Enum.filter(fn tx -> tx.value > 0 end) + |> Enum.map(fn tx -> + to_address_hash = Map.get(tx, :to_address_hash) || Map.get(tx, :created_contract_address_hash) + log_index = -1 * (tx.index + 1) * @transaction_buffer_size + {:ok, celo_token_address} = CeloCoreContracts.get_address(:celo_token, tx.block_number) + + %{ + amount: Decimal.new(tx.value), + block_hash: tx.block_hash, + block_number: tx.block_number, + from_address_hash: tx.from_address_hash, + log_index: log_index, + to_address_hash: to_address_hash, + token_contract_address_hash: celo_token_address, + token_ids: nil, + token_type: @token_type, + transaction_hash: tx.hash + } + end) + |> tap(&Logger.debug("Found #{length(&1)} Celo token transfers.")) + else + [] + end + + token_transfers + |> filter_tokens_for_supply_update() + |> TokenTotalSupplyUpdater.add_tokens() + + %{ + token_transfers: token_transfers, + tokens: to_tokens(token_transfers) + } + end + + def parse_internal_transactions(internal_transactions, block_number_to_block_hash) do + token_transfers = + internal_transactions + |> Enum.filter(fn itx -> + itx.value > 0 && + itx.index > 0 && + not Map.has_key?(itx, :error) && + (not Map.has_key?(itx, :call_type) || itx.call_type != "delegatecall") + end) + |> Enum.map(fn itx -> + to_address_hash = Map.get(itx, :to_address_hash) || Map.get(itx, :created_contract_address_hash) + log_index = -1 * (itx.transaction_index * @transaction_buffer_size + itx.index) + {:ok, celo_token_address} = CeloCoreContracts.get_address(:celo_token, itx.block_number) + + %{ + amount: Decimal.new(itx.value), + block_hash: block_number_to_block_hash[itx.block_number], + block_number: itx.block_number, + from_address_hash: itx.from_address_hash, + log_index: log_index, + to_address_hash: to_address_hash, + token_contract_address_hash: celo_token_address, + token_ids: nil, + token_type: @token_type, + transaction_hash: itx.transaction_hash + } + end) + + Logger.debug("Found #{length(token_transfers)} Celo token transfers from internal transactions.") + + token_transfers + |> filter_tokens_for_supply_update() + |> TokenTotalSupplyUpdater.add_tokens() + + %{ + token_transfers: token_transfers, + tokens: to_tokens(token_transfers) + } + end + + defp to_tokens([]), do: [] + + defp to_tokens(token_transfers) do + token_transfers + |> Enum.map( + &%{ + contract_address_hash: &1.token_contract_address_hash, + type: @token_type + } + ) + |> Enum.uniq() + end +end diff --git a/apps/indexer/lib/indexer/transform/celo/validator_epoch_payment_distributions.ex b/apps/indexer/lib/indexer/transform/celo/validator_epoch_payment_distributions.ex new file mode 100644 index 0000000000..69ace6db3b --- /dev/null +++ b/apps/indexer/lib/indexer/transform/celo/validator_epoch_payment_distributions.ex @@ -0,0 +1,72 @@ +defmodule Indexer.Transform.Celo.ValidatorEpochPaymentDistributions do + @moduledoc """ + Extracts data from `ValidatorEpochPaymentDistributed` event logs of the + `Validators` Celo core contract. + """ + alias ABI.FunctionSelector + + alias Explorer.Chain.Cache.CeloCoreContracts + alias Explorer.Chain.{Hash, Log} + + require Logger + + @event_signature "0x6f5937add2ec38a0fa4959bccd86e3fcc2aafb706cd3e6c0565f87a7b36b9975" + + @event_abi [ + %{ + "name" => "ValidatorEpochPaymentDistributed", + "type" => "event", + "anonymous" => false, + "inputs" => [ + %{ + "indexed" => true, + "name" => "validator", + "type" => "address" + }, + %{ + "indexed" => false, + "name" => "validatorPayment", + "type" => "uint256" + }, + %{ + "indexed" => true, + "name" => "group", + "type" => "address" + }, + %{ + "indexed" => false, + "name" => "groupPayment", + "type" => "uint256" + } + ] + } + ] + + def signature, do: @event_signature + + def parse(logs) do + logs + |> Enum.filter(fn log -> + {:ok, validators_contract_address} = CeloCoreContracts.get_address(:validators, log.block_number) + + Hash.to_string(log.address_hash) == validators_contract_address and + Hash.to_string(log.first_topic) == @event_signature + end) + |> Enum.map(fn log -> + {:ok, %FunctionSelector{}, + [ + {"validator", "address", true, validator_address}, + {"validatorPayment", "uint256", false, validator_payment}, + {"group", "address", true, group_address}, + {"groupPayment", "uint256", false, group_payment} + ]} = Log.find_and_decode(@event_abi, log, log.block_hash) + + %{ + validator_address: "0x" <> Base.encode16(validator_address, case: :lower), + validator_payment: validator_payment, + group_address: "0x" <> Base.encode16(group_address, case: :lower), + group_payment: group_payment + } + end) + end +end diff --git a/apps/indexer/lib/indexer/transform/token_transfers.ex b/apps/indexer/lib/indexer/transform/token_transfers.ex index 161e8dfdcb..8ac1af300c 100644 --- a/apps/indexer/lib/indexer/transform/token_transfers.ex +++ b/apps/indexer/lib/indexer/transform/token_transfers.ex @@ -63,14 +63,7 @@ defmodule Indexer.Transform.TokenTransfers do token_transfers = sanitize_weth_transfers(tokens, rough_token_transfers, weth_transfers.token_transfers) token_transfers - |> Enum.filter(fn token_transfer -> - token_transfer.to_address_hash == burn_address_hash_string() || - token_transfer.from_address_hash == burn_address_hash_string() - end) - |> Enum.map(fn token_transfer -> - token_transfer.token_contract_address_hash - end) - |> Enum.uniq() + |> filter_tokens_for_supply_update() |> TokenTotalSupplyUpdater.add_tokens() tokens_uniq = tokens |> Enum.uniq() @@ -484,6 +477,16 @@ defmodule Indexer.Transform.TokenTransfers do end end + def filter_tokens_for_supply_update(token_transfers) do + token_transfers + |> Enum.filter(fn token_transfer -> + token_transfer.to_address_hash == burn_address_hash_string() || + token_transfer.from_address_hash == burn_address_hash_string() + end) + |> Enum.map(& &1.token_contract_address_hash) + |> Enum.uniq() + end + defp encode_address_hash(binary) do "0x" <> Base.encode16(binary, case: :lower) end diff --git a/apps/indexer/test/indexer/block/catchup/bound_interval_supervisor_test.exs b/apps/indexer/test/indexer/block/catchup/bound_interval_supervisor_test.exs index 8d9650a762..fc81b00086 100644 --- a/apps/indexer/test/indexer/block/catchup/bound_interval_supervisor_test.exs +++ b/apps/indexer/test/indexer/block/catchup/bound_interval_supervisor_test.exs @@ -34,6 +34,8 @@ defmodule Indexer.Block.Catchup.BoundIntervalSupervisorTest do setup do initial_env = Application.get_env(:indexer, :block_ranges) on_exit(fn -> Application.put_env(:indexer, :block_ranges, initial_env) end) + + set_celo_core_contracts_env_var() end # See https://github.com/poanetwork/blockscout/issues/597 @@ -416,6 +418,9 @@ defmodule Indexer.Block.Catchup.BoundIntervalSupervisorTest do setup context do initial_env = Application.get_env(:indexer, :block_ranges) on_exit(fn -> Application.put_env(:indexer, :block_ranges, initial_env) end) + + set_celo_core_contracts_env_var() + # force to use `Mox`, so we can manipulate `latest_block_number` put_in(context.json_rpc_named_arguments[:transport], EthereumJSONRPC.Mox) end @@ -592,4 +597,28 @@ defmodule Indexer.Block.Catchup.BoundIntervalSupervisorTest do {:ok, %{pid: pid}} end + + defp set_celo_core_contracts_env_var do + Application.put_env(:explorer, Explorer.Chain.Cache.CeloCoreContracts, + contracts: %{ + "addresses" => %{ + "Accounts" => [], + "Election" => [], + "EpochRewards" => [], + "FeeHandler" => [], + "GasPriceMinimum" => [], + "GoldToken" => [], + "Governance" => [], + "LockedGold" => [], + "Reserve" => [], + "StableToken" => [], + "Validators" => [] + } + } + ) + + on_exit(fn -> + Application.put_env(:explorer, Explorer.Chain.Cache.CeloCoreContracts, contracts: %{}) + end) + end end diff --git a/apps/indexer/test/indexer/block/catchup/fetcher_test.exs b/apps/indexer/test/indexer/block/catchup/fetcher_test.exs index fcec1ef199..292b48183c 100644 --- a/apps/indexer/test/indexer/block/catchup/fetcher_test.exs +++ b/apps/indexer/test/indexer/block/catchup/fetcher_test.exs @@ -40,6 +40,7 @@ defmodule Indexer.Block.Catchup.FetcherTest do setup do configuration = Application.get_env(:indexer, :last_block) Application.put_env(:indexer, :last_block, 0) + Application.put_env(:indexer, Indexer.Fetcher.Celo.EpochBlockOperations.Supervisor, disabled?: true) on_exit(fn -> Application.put_env(:indexer, :last_block, configuration) @@ -141,6 +142,29 @@ defmodule Indexer.Block.Catchup.FetcherTest do setup do initial_env = Application.get_env(:indexer, :block_ranges) on_exit(fn -> Application.put_env(:indexer, :block_ranges, initial_env) end) + Application.put_env(:indexer, Indexer.Fetcher.Celo.EpochBlockOperations.Supervisor, disabled?: true) + + Application.put_env(:explorer, Explorer.Chain.Cache.CeloCoreContracts, + contracts: %{ + "addresses" => %{ + "Accounts" => [], + "Election" => [], + "EpochRewards" => [], + "FeeHandler" => [], + "GasPriceMinimum" => [], + "GoldToken" => [], + "Governance" => [], + "LockedGold" => [], + "Reserve" => [], + "StableToken" => [], + "Validators" => [] + } + } + ) + + on_exit(fn -> + Application.put_env(:explorer, Explorer.Chain.Cache.CeloCoreContracts, contracts: %{}) + end) end test "ignores fetched beneficiaries with different hash for same number", %{ @@ -617,7 +641,7 @@ defmodule Indexer.Block.Catchup.FetcherTest do MissingRangesManipulator.start_link([]) EthereumJSONRPC.Mox - |> expect(:json_rpc, 2, fn + |> expect(:json_rpc, 1, fn [ %{ id: id_1, @@ -646,9 +670,6 @@ defmodule Indexer.Block.Catchup.FetcherTest do error: %{message: "error"} } ]} - - [], _options -> - {:ok, []} end) Process.sleep(50) diff --git a/apps/indexer/test/indexer/block/fetcher_test.exs b/apps/indexer/test/indexer/block/fetcher_test.exs index ca9ae2a099..ce5ff1686a 100644 --- a/apps/indexer/test/indexer/block/fetcher_test.exs +++ b/apps/indexer/test/indexer/block/fetcher_test.exs @@ -60,6 +60,10 @@ defmodule Indexer.Block.FetcherTest do block_fetcher: %Fetcher{json_rpc_named_arguments: json_rpc_named_arguments} ) + Application.put_env(:indexer, Indexer.Fetcher.Celo.EpochBlockOperations.Supervisor, disabled?: true) + + maybe_set_celo_core_contracts_env() + %{ block_fetcher: %Fetcher{ broadcast: false, @@ -279,6 +283,31 @@ defmodule Indexer.Block.FetcherTest do to_address_hash = "0x8bf38d4764929064f2d4d3a56520a76ab3df415b" transaction_hash = "0x53bd884872de3e488692881baeec262e7b95234d3965248c39fe992fffd433e5" + transaction = %{ + "blockHash" => "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd", + "blockNumber" => "0x25", + "chainId" => "0x4d", + "condition" => nil, + "creates" => nil, + "from" => from_address_hash, + "gas" => "0x47b760", + "gasPrice" => "0x174876e800", + "hash" => transaction_hash, + "input" => "0x10855269000000000000000000000000862d67cb0773ee3f8ce7ea89b328ffea861ab3ef", + "nonce" => "0x4", + "publicKey" => + "0xe5d196ad4ceada719d9e592f7166d0c75700f6eab2e3c3de34ba751ea786527cb3f6eb96ad9fdfdb9989ff572df50f1c42ef800af9c5207a38b929aff969b5c9", + "r" => "0xa7f8f45cce375bb7af8750416e1b03e0473f93c256da2285d1134fc97a700e01", + "raw" => + "0xf88a0485174876e8008347b760948bf38d4764929064f2d4d3a56520a76ab3df415b80a410855269000000000000000000000000862d67cb0773ee3f8ce7ea89b328ffea861ab3ef81bea0a7f8f45cce375bb7af8750416e1b03e0473f93c256da2285d1134fc97a700e01a01f87a076f13824f4be8963e3dffd7300dae64d5f23c9a062af0c6ead347c135f", + "s" => "0x1f87a076f13824f4be8963e3dffd7300dae64d5f23c9a062af0c6ead347c135f", + "standardV" => "0x1", + "to" => to_address_hash, + "transactionIndex" => "0x0", + "v" => "0xbe", + "value" => "0x0" + } + EthereumJSONRPC.Mox |> expect(:json_rpc, fn json, _options -> assert [%{id: id, method: "eth_getBlockByNumber", params: [^block_quantity, true]}] = json @@ -313,32 +342,7 @@ defmodule Indexer.Block.FetcherTest do "step" => "302674398", "timestamp" => "0x5a343956", "totalDifficulty" => "0x24ffffffffffffffffffffffffedf78dfd", - "transactions" => [ - %{ - "blockHash" => "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd", - "blockNumber" => "0x25", - "chainId" => "0x4d", - "condition" => nil, - "creates" => nil, - "from" => from_address_hash, - "gas" => "0x47b760", - "gasPrice" => "0x174876e800", - "hash" => transaction_hash, - "input" => "0x10855269000000000000000000000000862d67cb0773ee3f8ce7ea89b328ffea861ab3ef", - "nonce" => "0x4", - "publicKey" => - "0xe5d196ad4ceada719d9e592f7166d0c75700f6eab2e3c3de34ba751ea786527cb3f6eb96ad9fdfdb9989ff572df50f1c42ef800af9c5207a38b929aff969b5c9", - "r" => "0xa7f8f45cce375bb7af8750416e1b03e0473f93c256da2285d1134fc97a700e01", - "raw" => - "0xf88a0485174876e8008347b760948bf38d4764929064f2d4d3a56520a76ab3df415b80a410855269000000000000000000000000862d67cb0773ee3f8ce7ea89b328ffea861ab3ef81bea0a7f8f45cce375bb7af8750416e1b03e0473f93c256da2285d1134fc97a700e01a01f87a076f13824f4be8963e3dffd7300dae64d5f23c9a062af0c6ead347c135f", - "s" => "0x1f87a076f13824f4be8963e3dffd7300dae64d5f23c9a062af0c6ead347c135f", - "standardV" => "0x1", - "to" => to_address_hash, - "transactionIndex" => "0x0", - "v" => "0xbe", - "value" => "0x0" - } - ], + "transactions" => [transaction], "transactionsRoot" => "0x68e314a05495f390f9cd0c36267159522e5450d2adf254a74567b452e767bf34", "uncles" => [] } @@ -433,32 +437,7 @@ defmodule Indexer.Block.FetcherTest do "step" => "302674398", "timestamp" => "0x5a343956", "totalDifficulty" => "0x24ffffffffffffffffffffffffedf78dfd", - "transactions" => [ - %{ - "blockHash" => "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd", - "blockNumber" => "0x25", - "chainId" => "0x4d", - "condition" => nil, - "creates" => nil, - "from" => from_address_hash, - "gas" => "0x47b760", - "gasPrice" => "0x174876e800", - "hash" => transaction_hash, - "input" => "0x10855269000000000000000000000000862d67cb0773ee3f8ce7ea89b328ffea861ab3ef", - "nonce" => "0x4", - "publicKey" => - "0xe5d196ad4ceada719d9e592f7166d0c75700f6eab2e3c3de34ba751ea786527cb3f6eb96ad9fdfdb9989ff572df50f1c42ef800af9c5207a38b929aff969b5c9", - "r" => "0xa7f8f45cce375bb7af8750416e1b03e0473f93c256da2285d1134fc97a700e01", - "raw" => - "0xf88a0485174876e8008347b760948bf38d4764929064f2d4d3a56520a76ab3df415b80a410855269000000000000000000000000862d67cb0773ee3f8ce7ea89b328ffea861ab3ef81bea0a7f8f45cce375bb7af8750416e1b03e0473f93c256da2285d1134fc97a700e01a01f87a076f13824f4be8963e3dffd7300dae64d5f23c9a062af0c6ead347c135f", - "s" => "0x1f87a076f13824f4be8963e3dffd7300dae64d5f23c9a062af0c6ead347c135f", - "standardV" => "0x1", - "to" => to_address_hash, - "transactionIndex" => "0x0", - "v" => "0xbe", - "value" => "0x0" - } - ], + "transactions" => [transaction], "transactionsRoot" => "0x68e314a05495f390f9cd0c36267159522e5450d2adf254a74567b452e767bf34", "uncles" => [] } @@ -705,88 +684,89 @@ defmodule Indexer.Block.FetcherTest do if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do EthereumJSONRPC.Mox - |> expect(:json_rpc, 2, fn requests, _options -> - {:ok, - Enum.map(requests, fn - %{id: id, method: "eth_getBlockByNumber", params: ["0x708677", true]} -> - %{ - id: id, - result: %{ - "author" => "0x5a0b54d5dc17e0aadc383d2db43b0a0d3e029c4c", - "difficulty" => "0x6bc767dd80781", - "extraData" => "0x5050594520737061726b706f6f6c2d6574682d7477", - "gasLimit" => "0x7a121d", - "gasUsed" => "0x79cbe9", - "hash" => "0x1b6fb99af0b51af6685a191b2f7bcba684f8565629bf084c70b2530479407455", - "logsBloom" => - "0x044d42d008801488400e1809190200a80d06105bc0c4100b047895c0d518327048496108388040140010b8208006288102e206160e21052322440924002090c1c808a0817405ab238086d028211014058e949401012403210314896702d06880c815c3060a0f0809987c81044488292cc11d57882c912a808ca10471c84460460040000c0001012804022000a42106591881d34407420ba401e1c08a8d00a000a34c11821a80222818a4102152c8a0c044032080c6462644223104d618e0e544072008120104408205c60510542264808488220403000106281a0290404220112c10b080145028c8000300b18a2c8280701c882e702210b00410834840108084", - "miner" => "0x5a0b54d5dc17e0aadc383d2db43b0a0d3e029c4c", - "mixHash" => "0xda53ae7c2b3c529783d6cdacdb90587fd70eb651c0f04253e8ff17de97844010", - "nonce" => "0x0946e5f01fce12bc", - "number" => "0x708677", - "parentHash" => "0x62543e836e0ef7edfa9e38f26526092c4be97efdf5ba9e0f53a4b0b7d5bc930a", - "receiptsRoot" => "0xa7d2b82bd8526de11736c18bd5cc8cfe2692106c4364526f3310ad56d78669c4", - "sealFields" => [ - "0xa0da53ae7c2b3c529783d6cdacdb90587fd70eb651c0f04253e8ff17de97844010", - "0x880946e5f01fce12bc" - ], - "sha3Uncles" => "0x483a8a21a5825ad270f358b3ea56e060bbb8b3082d9a92ec8fa17a5c7e6fc1b6", - "size" => "0x544c", - "stateRoot" => "0x85daa9cd528004c1609d4cb3520fd958e85983bb4183124a4a9f7137fd39c691", - "timestamp" => "0x5c8bc76e", - "totalDifficulty" => "0x201a42c35142ae94458", - "transactions" => [], - "transactionsRoot" => "0xcd6c12fa43cd4e92ad5c0bf232b30488bbcbfe273c5b4af0366fced0767d54db", - "uncles" => [] + |> expect(:json_rpc, 2, fn + requests, _options -> + {:ok, + Enum.map(requests, fn + %{id: id, method: "eth_getBlockByNumber", params: ["0x708677", true]} -> + %{ + id: id, + result: %{ + "author" => "0x5a0b54d5dc17e0aadc383d2db43b0a0d3e029c4c", + "difficulty" => "0x6bc767dd80781", + "extraData" => "0x5050594520737061726b706f6f6c2d6574682d7477", + "gasLimit" => "0x7a121d", + "gasUsed" => "0x79cbe9", + "hash" => "0x1b6fb99af0b51af6685a191b2f7bcba684f8565629bf084c70b2530479407455", + "logsBloom" => + "0x044d42d008801488400e1809190200a80d06105bc0c4100b047895c0d518327048496108388040140010b8208006288102e206160e21052322440924002090c1c808a0817405ab238086d028211014058e949401012403210314896702d06880c815c3060a0f0809987c81044488292cc11d57882c912a808ca10471c84460460040000c0001012804022000a42106591881d34407420ba401e1c08a8d00a000a34c11821a80222818a4102152c8a0c044032080c6462644223104d618e0e544072008120104408205c60510542264808488220403000106281a0290404220112c10b080145028c8000300b18a2c8280701c882e702210b00410834840108084", + "miner" => "0x5a0b54d5dc17e0aadc383d2db43b0a0d3e029c4c", + "mixHash" => "0xda53ae7c2b3c529783d6cdacdb90587fd70eb651c0f04253e8ff17de97844010", + "nonce" => "0x0946e5f01fce12bc", + "number" => "0x708677", + "parentHash" => "0x62543e836e0ef7edfa9e38f26526092c4be97efdf5ba9e0f53a4b0b7d5bc930a", + "receiptsRoot" => "0xa7d2b82bd8526de11736c18bd5cc8cfe2692106c4364526f3310ad56d78669c4", + "sealFields" => [ + "0xa0da53ae7c2b3c529783d6cdacdb90587fd70eb651c0f04253e8ff17de97844010", + "0x880946e5f01fce12bc" + ], + "sha3Uncles" => "0x483a8a21a5825ad270f358b3ea56e060bbb8b3082d9a92ec8fa17a5c7e6fc1b6", + "size" => "0x544c", + "stateRoot" => "0x85daa9cd528004c1609d4cb3520fd958e85983bb4183124a4a9f7137fd39c691", + "timestamp" => "0x5c8bc76e", + "totalDifficulty" => "0x201a42c35142ae94458", + "transactions" => [], + "transactionsRoot" => "0xcd6c12fa43cd4e92ad5c0bf232b30488bbcbfe273c5b4af0366fced0767d54db", + "uncles" => [] + } } - } - %{id: id, method: "trace_block"} -> - block_quantity = integer_to_quantity(block_number) - _res = eth_block_number_fake_response(block_quantity) + %{id: id, method: "trace_block"} -> + block_quantity = integer_to_quantity(block_number) + _res = eth_block_number_fake_response(block_quantity) - %{ - id: id, - result: [ - %{ - "action" => %{ - "author" => "0x5a0b54d5dc17e0aadc383d2db43b0a0d3e029c4c", - "rewardType" => "block", - "value" => "0x1d7d843dc3b48000" - }, - "blockHash" => "0x1b6fb99af0b51af6685a191b2f7bcba684f8565629bf084c70b2530479407455", - "blockNumber" => block_number, - "subtraces" => 0, - "traceAddress" => [], - "type" => "reward" - }, - %{ - "action" => %{ - "author" => "0xea674fdde714fd979de3edf0f56aa9716b898ec8", - "rewardType" => "uncle", - "value" => "0x14d1120d7b160000" + %{ + id: id, + result: [ + %{ + "action" => %{ + "author" => "0x5a0b54d5dc17e0aadc383d2db43b0a0d3e029c4c", + "rewardType" => "block", + "value" => "0x1d7d843dc3b48000" + }, + "blockHash" => "0x1b6fb99af0b51af6685a191b2f7bcba684f8565629bf084c70b2530479407455", + "blockNumber" => block_number, + "subtraces" => 0, + "traceAddress" => [], + "type" => "reward" }, - "blockHash" => "0x1b6fb99af0b51af6685a191b2f7bcba684f8565629bf084c70b2530479407455", - "blockNumber" => block_number, - "subtraces" => 0, - "traceAddress" => [], - "type" => "reward" - }, - %{ - "action" => %{ - "author" => "0xea674fdde714fd979de3edf0f56aa9716b898ec8", - "rewardType" => "uncle", - "value" => "0x18493fba64ef0000" + %{ + "action" => %{ + "author" => "0xea674fdde714fd979de3edf0f56aa9716b898ec8", + "rewardType" => "uncle", + "value" => "0x14d1120d7b160000" + }, + "blockHash" => "0x1b6fb99af0b51af6685a191b2f7bcba684f8565629bf084c70b2530479407455", + "blockNumber" => block_number, + "subtraces" => 0, + "traceAddress" => [], + "type" => "reward" }, - "blockHash" => "0x1b6fb99af0b51af6685a191b2f7bcba684f8565629bf084c70b2530479407455", - "blockNumber" => block_number, - "subtraces" => 0, - "traceAddress" => [], - "type" => "reward" - } - ] - } - end)} + %{ + "action" => %{ + "author" => "0xea674fdde714fd979de3edf0f56aa9716b898ec8", + "rewardType" => "uncle", + "value" => "0x18493fba64ef0000" + }, + "blockHash" => "0x1b6fb99af0b51af6685a191b2f7bcba684f8565629bf084c70b2530479407455", + "blockNumber" => block_number, + "subtraces" => 0, + "traceAddress" => [], + "type" => "reward" + } + ] + } + end)} end) end @@ -797,6 +777,387 @@ defmodule Indexer.Block.FetcherTest do end end + if Application.compile_env(:explorer, :chain_type) == :celo do + describe "import_range/2 celo" do + setup %{json_rpc_named_arguments: json_rpc_named_arguments} do + CoinBalanceCatchup.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) + ContractCode.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) + InternalTransaction.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) + Token.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) + TokenBalance.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) + ReplacedTransaction.Supervisor.Case.start_supervised!() + + UncleBlock.Supervisor.Case.start_supervised!( + block_fetcher: %Fetcher{json_rpc_named_arguments: json_rpc_named_arguments} + ) + + Application.put_env(:indexer, Indexer.Fetcher.Celo.EpochBlockOperations.Supervisor, disabled?: true) + + maybe_set_celo_core_contracts_env() + + %{ + block_fetcher: %Fetcher{ + broadcast: false, + callback_module: Indexer.Block.Catchup.Fetcher, + json_rpc_named_arguments: json_rpc_named_arguments + } + } + end + + test "can import range with all synchronous imported schemas", %{ + block_fetcher: %Fetcher{json_rpc_named_arguments: json_rpc_named_arguments} = block_fetcher + } do + block_number = @first_full_block_number + + if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do + case Keyword.fetch!(json_rpc_named_arguments, :variant) do + EthereumJSONRPC.Nethermind -> + block_quantity = integer_to_quantity(block_number) + from_address_hash = "0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca" + to_address_hash = "0x8bf38d4764929064f2d4d3a56520a76ab3df415b" + transaction_hash = "0x53bd884872de3e488692881baeec262e7b95234d3965248c39fe992fffd433e5" + gas_token_contract_address_hash = "0x471ece3750da237f93b8e339c536989b8978a438" + + transaction = %{ + "blockHash" => "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd", + "blockNumber" => "0x25", + "chainId" => "0x4d", + "condition" => nil, + "creates" => nil, + "from" => from_address_hash, + "gas" => "0x47b760", + "gasPrice" => "0x174876e800", + "hash" => transaction_hash, + "input" => "0x10855269000000000000000000000000862d67cb0773ee3f8ce7ea89b328ffea861ab3ef", + "nonce" => "0x4", + "publicKey" => + "0xe5d196ad4ceada719d9e592f7166d0c75700f6eab2e3c3de34ba751ea786527cb3f6eb96ad9fdfdb9989ff572df50f1c42ef800af9c5207a38b929aff969b5c9", + "r" => "0xa7f8f45cce375bb7af8750416e1b03e0473f93c256da2285d1134fc97a700e01", + "raw" => + "0xf88a0485174876e8008347b760948bf38d4764929064f2d4d3a56520a76ab3df415b80a410855269000000000000000000000000862d67cb0773ee3f8ce7ea89b328ffea861ab3ef81bea0a7f8f45cce375bb7af8750416e1b03e0473f93c256da2285d1134fc97a700e01a01f87a076f13824f4be8963e3dffd7300dae64d5f23c9a062af0c6ead347c135f", + "s" => "0x1f87a076f13824f4be8963e3dffd7300dae64d5f23c9a062af0c6ead347c135f", + "standardV" => "0x1", + "to" => to_address_hash, + "transactionIndex" => "0x0", + "v" => "0xbe", + "value" => "0x0", + # Celo-specific fields + "feeCurrency" => gas_token_contract_address_hash, + "gatewayFeeRecipient" => nil, + "gatewayFee" => "0x0", + "ethCompatible" => false + } + + EthereumJSONRPC.Mox + |> expect(:json_rpc, fn json, _options -> + assert [%{id: id, method: "eth_getBlockByNumber", params: [^block_quantity, true]}] = json + + {:ok, + [ + %{ + id: id, + jsonrpc: "2.0", + result: %{ + "author" => "0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca", + "difficulty" => "0xfffffffffffffffffffffffffffffffe", + "extraData" => "0xd5830108048650617269747986312e32322e31826c69", + "gasLimit" => "0x69fe20", + "gasUsed" => "0xc512", + "hash" => "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd", + "logsBloom" => + "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000400000000000000000000000000000000000000000000000200000000000000000000020000000000000000200000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "miner" => "0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca", + "number" => "0x25", + "parentHash" => "0xc37bbad7057945d1bf128c1ff009fb1ad632110bf6a000aac025a80f7766b66e", + "receiptsRoot" => "0xd300311aab7dcc98c05ac3f1893629b2c9082c189a0a0c76f4f63e292ac419d5", + "sealFields" => [ + "0x84120a71de", + "0xb841fcdb570511ec61edda93849bb7c6b3232af60feb2ea74e4035f0143ab66dfdd00f67eb3eda1adddbb6b572db1e0abd39ce00f9b3ccacb9f47973279ff306fe5401" + ], + "sha3Uncles" => "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "signature" => + "fcdb570511ec61edda93849bb7c6b3232af60feb2ea74e4035f0143ab66dfdd00f67eb3eda1adddbb6b572db1e0abd39ce00f9b3ccacb9f47973279ff306fe5401", + "size" => "0x2cf", + "stateRoot" => "0x2cd84079b0d0c267ed387e3895fd1c1dc21ff82717beb1132adac64276886e19", + "step" => "302674398", + "timestamp" => "0x5a343956", + "totalDifficulty" => "0x24ffffffffffffffffffffffffedf78dfd", + "transactions" => [transaction], + "transactionsRoot" => "0x68e314a05495f390f9cd0c36267159522e5450d2adf254a74567b452e767bf34", + "uncles" => [] + } + } + ]} + end) + |> expect(:json_rpc, fn json, _options -> + assert [ + %{ + id: id, + method: "eth_getTransactionReceipt", + params: ["0x53bd884872de3e488692881baeec262e7b95234d3965248c39fe992fffd433e5"] + } + ] = json + + {:ok, + [ + %{ + id: id, + jsonrpc: "2.0", + result: %{ + "blockHash" => "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd", + "blockNumber" => "0x25", + "contractAddress" => nil, + "cumulativeGasUsed" => "0xc512", + "gasUsed" => "0xc512", + "logs" => [ + %{ + "address" => "0x8bf38d4764929064f2d4d3a56520a76ab3df415b", + "blockHash" => "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd", + "blockNumber" => "0x25", + "data" => "0x000000000000000000000000862d67cb0773ee3f8ce7ea89b328ffea861ab3ef", + "logIndex" => "0x0", + "topics" => ["0x600bcf04a13e752d1e3670a5a9f1c21177ca2a93c6f5391d4f1298d098097c22"], + "transactionHash" => "0x53bd884872de3e488692881baeec262e7b95234d3965248c39fe992fffd433e5", + "transactionIndex" => "0x0", + "transactionLogIndex" => "0x0" + } + ], + "logsBloom" => + "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000400000000000000000000000000000000000000000000000200000000000000000000020000000000000000200000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "root" => nil, + "status" => "0x1", + "transactionHash" => "0x53bd884872de3e488692881baeec262e7b95234d3965248c39fe992fffd433e5", + "transactionIndex" => "0x0" + } + } + ]} + end) + |> expect(:json_rpc, fn [%{id: id, method: "trace_block", params: [^block_quantity]}], _options -> + {:ok, [%{id: id, result: []}]} + end) + # async requests need to be grouped in one expect because the order is non-deterministic while multiple expect + # calls on the same name/arity are used in order + |> expect(:json_rpc, 10, fn json, _options -> + case json do + [ + %{ + id: 0, + jsonrpc: "2.0", + method: "eth_getBlockByNumber", + params: [^block_quantity, true] + } + ] -> + {:ok, + [ + %{ + id: 0, + jsonrpc: "2.0", + result: %{ + "author" => "0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca", + "difficulty" => "0xfffffffffffffffffffffffffffffffe", + "extraData" => "0xd5830108048650617269747986312e32322e31826c69", + "gasLimit" => "0x69fe20", + "gasUsed" => "0xc512", + "hash" => "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd", + "logsBloom" => + "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000400000000000000000000000000000000000000000000000200000000000000000000020000000000000000200000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "miner" => "0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca", + "number" => "0x25", + "parentHash" => "0xc37bbad7057945d1bf128c1ff009fb1ad632110bf6a000aac025a80f7766b66e", + "receiptsRoot" => "0xd300311aab7dcc98c05ac3f1893629b2c9082c189a0a0c76f4f63e292ac419d5", + "sealFields" => [ + "0x84120a71de", + "0xb841fcdb570511ec61edda93849bb7c6b3232af60feb2ea74e4035f0143ab66dfdd00f67eb3eda1adddbb6b572db1e0abd39ce00f9b3ccacb9f47973279ff306fe5401" + ], + "sha3Uncles" => "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "signature" => + "fcdb570511ec61edda93849bb7c6b3232af60feb2ea74e4035f0143ab66dfdd00f67eb3eda1adddbb6b572db1e0abd39ce00f9b3ccacb9f47973279ff306fe5401", + "size" => "0x2cf", + "stateRoot" => "0x2cd84079b0d0c267ed387e3895fd1c1dc21ff82717beb1132adac64276886e19", + "step" => "302674398", + "timestamp" => "0x5a343956", + "totalDifficulty" => "0x24ffffffffffffffffffffffffedf78dfd", + "transactions" => [transaction], + "transactionsRoot" => "0x68e314a05495f390f9cd0c36267159522e5450d2adf254a74567b452e767bf34", + "uncles" => [] + } + } + ]} + + [%{id: id, method: "eth_getBalance", params: [^to_address_hash, ^block_quantity]}] -> + {:ok, [%{id: id, jsonrpc: "2.0", result: "0x1"}]} + + [%{id: id, method: "eth_getBalance", params: [^from_address_hash, ^block_quantity]}] -> + {:ok, [%{id: id, jsonrpc: "2.0", result: "0xd0d4a965ab52d8cd740000"}]} + + [%{id: id, method: "trace_replayBlockTransactions", params: [^block_quantity, ["trace"]]}] -> + {:ok, + [ + %{ + id: id, + jsonrpc: "2.0", + result: [ + %{ + "output" => "0x", + "stateDiff" => nil, + "trace" => [ + %{ + "action" => %{ + "callType" => "call", + "from" => from_address_hash, + "gas" => "0x475ec8", + "input" => + "0x10855269000000000000000000000000862d67cb0773ee3f8ce7ea89b328ffea861ab3ef", + "to" => to_address_hash, + "value" => "0x0" + }, + "result" => %{"gasUsed" => "0x6c7a", "output" => "0x"}, + "subtraces" => 0, + "traceAddress" => [], + "type" => "call" + } + ], + "transactionHash" => transaction_hash, + "vmTrace" => nil + } + ] + } + ]} + + requests -> + {:ok, + Enum.map(requests, fn + %{id: id, method: "eth_call", params: [%{data: "0x313ce567", to: _}, "latest"]} -> + %{ + id: id, + result: "0x0000000000000000000000000000000000000000000000000000000000000012" + } + + %{id: id, method: "eth_call", params: [%{data: "0x06fdde03", to: _}, "latest"]} -> + %{ + id: id, + result: + "0x0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000642616e636f720000000000000000000000000000000000000000000000000000" + } + + %{id: id, method: "eth_call", params: [%{data: "0x95d89b41", to: _}, "latest"]} -> + %{ + id: id, + result: + "0x00000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000003424e540000000000000000000000000000000000000000000000000000000000" + } + + %{id: id, method: "eth_call", params: [%{data: "0x18160ddd", to: _}, "latest"]} -> + %{ + id: id, + result: "0x0000000000000000000000000000000000000000000000000de0b6b3a7640000" + } + end)} + end + end) + + variant -> + raise ArgumentError, "Unsupported variant (#{variant})" + end + end + + case Keyword.fetch!(json_rpc_named_arguments, :variant) do + EthereumJSONRPC.Nethermind -> + gateway_fee_value = Decimal.new(0) + + assert {:ok, + %{ + inserted: %{ + addresses: [ + %Address{ + hash: + %Explorer.Chain.Hash{ + byte_count: 20, + bytes: + <<139, 243, 141, 71, 100, 146, 144, 100, 242, 212, 211, 165, 101, 32, 167, 106, 179, + 223, 65, 91>> + } = first_address_hash + }, + %Address{ + hash: + %Explorer.Chain.Hash{ + byte_count: 20, + bytes: + <<232, 221, 197, 199, 162, 210, 240, 215, 169, 121, 132, 89, 192, 16, 79, 223, 94, + 152, 122, 202>> + } = second_address_hash + } + ], + blocks: [ + %Chain.Block{ + hash: %Explorer.Chain.Hash{ + byte_count: 32, + bytes: + <<246, 180, 184, 200, 141, 243, 235, 210, 82, 236, 71, 99, 40, 51, 77, 192, 38, 207, + 102, 96, 106, 132, 251, 118, 155, 61, 60, 188, 204, 132, 113, 189>> + } + } + ], + logs: [ + %Log{ + index: 0, + transaction_hash: %Explorer.Chain.Hash{ + byte_count: 32, + bytes: + <<83, 189, 136, 72, 114, 222, 62, 72, 134, 146, 136, 27, 174, 236, 38, 46, 123, 149, 35, + 77, 57, 101, 36, 140, 57, 254, 153, 47, 255, 212, 51, 229>> + } + } + ], + transactions: [ + %Transaction{ + block_number: block_number, + index: 0, + hash: %Explorer.Chain.Hash{ + byte_count: 32, + bytes: + <<83, 189, 136, 72, 114, 222, 62, 72, 134, 146, 136, 27, 174, 236, 38, 46, 123, 149, 35, + 77, 57, 101, 36, 140, 57, 254, 153, 47, 255, 212, 51, 229>> + }, + gas_token_contract_address_hash: %Explorer.Chain.Hash{ + byte_count: 20, + bytes: + <<71, 30, 206, 55, 80, 218, 35, 127, 147, 184, 227, 57, 197, 54, 152, 155, 137, 120, + 164, 56>> + }, + gas_fee_recipient_address_hash: nil, + gateway_fee: %Explorer.Chain.Wei{value: ^gateway_fee_value} + } + ] + }, + errors: [] + }} = Fetcher.fetch_and_import_range(block_fetcher, block_number..block_number) + + wait_for_tasks(InternalTransaction) + wait_for_tasks(CoinBalanceCatchup) + + assert Repo.aggregate(Chain.Block, :count, :hash) == 1 + assert Repo.aggregate(Address, :count, :hash) == 2 + assert Chain.log_count() == 1 + assert Repo.aggregate(Transaction, :count, :hash) == 1 + + first_address = Repo.get!(Address, first_address_hash) + + assert first_address.fetched_coin_balance == %Wei{value: Decimal.new(1)} + assert first_address.fetched_coin_balance_block_number == block_number + + second_address = Repo.get!(Address, second_address_hash) + + assert second_address.fetched_coin_balance == %Wei{value: Decimal.new(252_460_837_000_000_000_000_000_000)} + assert second_address.fetched_coin_balance_block_number == block_number + + variant -> + raise ArgumentError, "Unsupported variant (#{variant})" + end + end + end + end + defp wait_until(timeout, producer) do parent = self() ref = make_ref() @@ -861,4 +1222,28 @@ defmodule Indexer.Block.FetcherTest do } } end + + def maybe_set_celo_core_contracts_env do + Application.put_env(:explorer, Explorer.Chain.Cache.CeloCoreContracts, + contracts: %{ + "addresses" => %{ + "Accounts" => [], + "Election" => [], + "EpochRewards" => [], + "FeeHandler" => [], + "GasPriceMinimum" => [], + "GoldToken" => [], + "Governance" => [], + "LockedGold" => [], + "Reserve" => [], + "StableToken" => [], + "Validators" => [] + } + } + ) + + on_exit(fn -> + Application.put_env(:explorer, Explorer.Chain.Cache.CeloCoreContracts, contracts: %{}) + end) + end end diff --git a/apps/indexer/test/indexer/block/realtime/fetcher_test.exs b/apps/indexer/test/indexer/block/realtime/fetcher_test.exs index 71b671ee27..12e164b634 100644 --- a/apps/indexer/test/indexer/block/realtime/fetcher_test.exs +++ b/apps/indexer/test/indexer/block/realtime/fetcher_test.exs @@ -4,11 +4,19 @@ defmodule Indexer.Block.Realtime.FetcherTest do import Mox - alias Explorer.Chain + alias Explorer.{Chain, Factory} alias Explorer.Chain.{Address, Transaction, Wei} alias Indexer.Block.Realtime alias Indexer.Fetcher.CoinBalance.Realtime, as: CoinBalanceRealtime - alias Indexer.Fetcher.{ContractCode, InternalTransaction, ReplacedTransaction, Token, TokenBalance, UncleBlock} + + alias Indexer.Fetcher.{ + ContractCode, + InternalTransaction, + ReplacedTransaction, + Token, + TokenBalance, + UncleBlock + } @moduletag capture_log: true @@ -38,6 +46,30 @@ defmodule Indexer.Block.Realtime.FetcherTest do TokenBalance.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) CoinBalanceRealtime.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) + Application.put_env(:indexer, Indexer.Fetcher.Celo.EpochBlockOperations.Supervisor, disabled?: true) + + Application.put_env(:explorer, Explorer.Chain.Cache.CeloCoreContracts, + contracts: %{ + "addresses" => %{ + "Accounts" => [], + "Election" => [], + "EpochRewards" => [], + "FeeHandler" => [], + "GasPriceMinimum" => [], + "GoldToken" => [], + "Governance" => [], + "LockedGold" => [], + "Reserve" => [], + "StableToken" => [], + "Validators" => [] + } + } + ) + + on_exit(fn -> + Application.put_env(:explorer, Explorer.Chain.Cache.CeloCoreContracts, contracts: %{}) + end) + %{block_fetcher: block_fetcher, json_rpc_named_arguments: core_json_rpc_named_arguments} end @@ -59,6 +91,36 @@ defmodule Indexer.Block.Realtime.FetcherTest do ReplacedTransaction.Supervisor.Case.start_supervised!() + # In CELO network, there is a token duality feature where CELO can be used + # as both a native chain currency and as an ERC-20 token (GoldToken). + # Transactions that transfer CELO are also counted as token transfers, and + # the TokenInstance fetcher is called. However, for simplicity, we disable + # it in this test. + Application.put_env(:indexer, Indexer.Fetcher.TokenInstance.Realtime.Supervisor, disabled?: true) + + on_exit(fn -> + Application.put_env(:indexer, Indexer.Fetcher.TokenInstance.Realtime.Supervisor, disabled?: false) + end) + + celo_token_address_hash = Factory.address_hash() + + Application.put_env(:explorer, Explorer.Chain.Cache.CeloCoreContracts, + contracts: %{ + "addresses" => %{ + "GoldToken" => [ + %{ + "address" => to_string(celo_token_address_hash), + "updated_at_block_number" => 3_946_079 + } + ] + } + } + ) + + on_exit(fn -> + Application.put_env(:explorer, Explorer.Chain.Cache.CeloCoreContracts, contracts: %{}) + end) + if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do EthereumJSONRPC.Mox |> expect(:json_rpc, fn [ @@ -474,12 +536,7 @@ defmodule Indexer.Block.Realtime.FetcherTest do assert {:ok, %{ inserted: %{ - addresses: [ - %Address{hash: first_address_hash}, - %Address{hash: second_address_hash}, - %Address{hash: third_address_hash}, - %Address{hash: fourth_address_hash} - ], + addresses: addresses, address_coin_balances: [ %{ address_hash: first_address_hash, @@ -503,6 +560,23 @@ defmodule Indexer.Block.Realtime.FetcherTest do }, errors: [] }} = Indexer.Block.Fetcher.fetch_and_import_range(block_fetcher, 3_946_079..3_946_080) + + unless Application.get_env(:explorer, :chain_type) == :celo do + assert [ + %Address{hash: ^first_address_hash}, + %Address{hash: ^second_address_hash}, + %Address{hash: ^third_address_hash}, + %Address{hash: ^fourth_address_hash} + ] = addresses + else + assert [ + %Address{hash: ^celo_token_address_hash}, + %Address{hash: ^first_address_hash}, + %Address{hash: ^second_address_hash}, + %Address{hash: ^third_address_hash}, + %Address{hash: ^fourth_address_hash} + ] = addresses + end end @tag :no_geth @@ -524,6 +598,36 @@ defmodule Indexer.Block.Realtime.FetcherTest do ReplacedTransaction.Supervisor.Case.start_supervised!() + # In CELO network, there is a token duality feature where CELO can be used + # as both a native chain currency and as an ERC-20 token (GoldToken). + # Transactions that transfer CELO are also counted as token transfers, and + # the TokenInstance fetcher is called. However, for simplicity, we disable + # it in this test. + Application.put_env(:indexer, Indexer.Fetcher.TokenInstance.Realtime.Supervisor, disabled?: true) + + on_exit(fn -> + Application.put_env(:indexer, Indexer.Fetcher.TokenInstance.Realtime.Supervisor, disabled?: false) + end) + + celo_token_address_hash = Factory.address_hash() + + Application.put_env(:explorer, Explorer.Chain.Cache.CeloCoreContracts, + contracts: %{ + "addresses" => %{ + "GoldToken" => [ + %{ + "address" => to_string(celo_token_address_hash), + "updated_at_block_number" => 3_946_079 + } + ] + } + } + ) + + on_exit(fn -> + Application.put_env(:explorer, Explorer.Chain.Cache.CeloCoreContracts, contracts: %{}) + end) + if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do EthereumJSONRPC.Mox |> expect(:json_rpc, fn [ @@ -676,12 +780,7 @@ defmodule Indexer.Block.Realtime.FetcherTest do assert {:ok, %{ inserted: %{ - addresses: [ - %Address{hash: first_address_hash}, - %Address{hash: second_address_hash}, - %Address{hash: third_address_hash}, - %Address{hash: fourth_address_hash} - ], + addresses: addresses, address_coin_balances: [ %{ address_hash: first_address_hash, @@ -718,6 +817,23 @@ defmodule Indexer.Block.Realtime.FetcherTest do errors: [] }} = Indexer.Block.Fetcher.fetch_and_import_range(block_fetcher, 3_946_079..3_946_080) + unless Application.get_env(:explorer, :chain_type) == :celo do + assert [ + %Address{hash: ^first_address_hash}, + %Address{hash: ^second_address_hash}, + %Address{hash: ^third_address_hash}, + %Address{hash: ^fourth_address_hash} + ] = addresses + else + assert [ + %Address{hash: ^celo_token_address_hash}, + %Address{hash: ^first_address_hash}, + %Address{hash: ^second_address_hash}, + %Address{hash: ^third_address_hash}, + %Address{hash: ^fourth_address_hash} + ] = addresses + end + Application.put_env(:indexer, :fetch_rewards_way, nil) end end diff --git a/apps/indexer/test/indexer/fetcher/celo/epoch_block_operations_test.exs b/apps/indexer/test/indexer/fetcher/celo/epoch_block_operations_test.exs new file mode 100644 index 0000000000..d812cc92d3 --- /dev/null +++ b/apps/indexer/test/indexer/fetcher/celo/epoch_block_operations_test.exs @@ -0,0 +1,46 @@ +defmodule Indexer.Fetcher.Celo.EpochBlockOperationsTest do + # MUST be `async: false` so that {:shared, pid} is set for connection to allow CoinBalanceFetcher's self-send to have + # connection allowed immediately. + # use EthereumJSONRPC.Case, async: false + + use EthereumJSONRPC.Case + use Explorer.DataCase + + import Mox + # import EthereumJSONRPC, only: [integer_to_quantity: 1] + + import Explorer.Chain.Celo.Helper, only: [blocks_per_epoch: 0] + + alias Indexer.Fetcher.Celo.EpochBlockOperations + + # @moduletag :capture_log + + # MUST use global mode because we aren't guaranteed to get `start_supervised`'s pid back fast enough to `allow` it to + # use expectations and stubs from test's pid. + setup :set_mox_global + + setup :verify_on_exit! + + if Application.compile_env(:explorer, :chain_type) == :celo do + describe "init/3" do + test "buffers blocks with pending epoch operation", %{ + json_rpc_named_arguments: json_rpc_named_arguments + } do + unfetched = insert(:block, number: 1 * blocks_per_epoch()) + insert(:celo_pending_epoch_block_operation, block: unfetched) + + assert [ + %{ + block_number: unfetched.number, + block_hash: unfetched.hash + } + ] == + EpochBlockOperations.init( + [], + fn block_number, acc -> [block_number | acc] end, + json_rpc_named_arguments + ) + end + end + end +end diff --git a/apps/indexer/test/indexer/fetcher/internal_transaction_test.exs b/apps/indexer/test/indexer/fetcher/internal_transaction_test.exs index fada8686e3..66de4d813a 100644 --- a/apps/indexer/test/indexer/fetcher/internal_transaction_test.exs +++ b/apps/indexer/test/indexer/fetcher/internal_transaction_test.exs @@ -10,7 +10,7 @@ defmodule Indexer.Fetcher.InternalTransactionTest do alias Explorer.Chain.{Block, PendingBlockOperation} alias Explorer.Chain.Import.Runner.Blocks alias Indexer.Fetcher.CoinBalance.Catchup, as: CoinBalanceCatchup - alias Indexer.Fetcher.{InternalTransaction, PendingTransaction} + alias Indexer.Fetcher.{InternalTransaction, PendingTransaction, TokenBalance} # MUST use global mode because we aren't guaranteed to get PendingTransactionFetcher's pid back fast enough to `allow` # it to use expectations and stubs from test's pid. @@ -68,6 +68,7 @@ defmodule Indexer.Fetcher.InternalTransactionTest do CoinBalanceCatchup.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) PendingTransaction.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) + start_token_balance_fetcher(json_rpc_named_arguments) wait_for_results(fn -> Repo.one!(from(transaction in Explorer.Chain.Transaction, where: is_nil(transaction.block_hash), limit: 1)) @@ -106,6 +107,8 @@ defmodule Indexer.Fetcher.InternalTransactionTest do block = insert(:block, number: block_number) insert(:pending_block_operation, block_hash: block.hash, block_number: block.number) + start_token_balance_fetcher(json_rpc_named_arguments) + assert :ok = InternalTransaction.run([block_number], json_rpc_named_arguments) assert InternalTransaction.init( @@ -174,6 +177,8 @@ defmodule Indexer.Fetcher.InternalTransactionTest do block_hash = block.hash insert(:pending_block_operation, block_hash: block_hash, block_number: block.number) + start_token_balance_fetcher(json_rpc_named_arguments) + assert %{block_hash: block_hash} = Repo.get(PendingBlockOperation, block_hash) assert :ok == InternalTransaction.run([block.number], json_rpc_named_arguments) @@ -278,6 +283,7 @@ defmodule Indexer.Fetcher.InternalTransactionTest do end CoinBalanceCatchup.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) + start_token_balance_fetcher(json_rpc_named_arguments) assert %{block_hash: block_hash} = Repo.get(PendingBlockOperation, block_hash) @@ -608,4 +614,15 @@ defmodule Indexer.Fetcher.InternalTransactionTest do assert last_int_tx.call_type == :invalid end end + + # Due to token-duality feature in Celo network (native coin transfers are + # treated as token transfers), we need to fetch updated token balances after + # parsing the internal transactions + if Application.compile_env(:explorer, :chain_type) == :celo do + defp start_token_balance_fetcher(json_rpc_named_arguments) do + TokenBalance.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) + end + else + defp start_token_balance_fetcher(_json_rpc_named_arguments), do: :ok + end end diff --git a/apps/indexer/test/support/indexer/fetcher/celo_epoch_rewards_supervisor_case.ex b/apps/indexer/test/support/indexer/fetcher/celo_epoch_rewards_supervisor_case.ex new file mode 100644 index 0000000000..507981237d --- /dev/null +++ b/apps/indexer/test/support/indexer/fetcher/celo_epoch_rewards_supervisor_case.ex @@ -0,0 +1,17 @@ +defmodule Indexer.Fetcher.Celo.EpochBlockOperations.Supervisor.Case do + alias Indexer.Fetcher.Celo.EpochBlockOperations + + def start_supervised!(fetcher_arguments \\ []) when is_list(fetcher_arguments) do + merged_fetcher_arguments = + Keyword.merge( + fetcher_arguments, + flush_interval: 50, + max_batch_size: 1, + max_concurrency: 1 + ) + + [merged_fetcher_arguments] + |> EpochBlockOperations.Supervisor.child_spec() + |> ExUnit.Callbacks.start_supervised!() + end +end diff --git a/config/config_helper.exs b/config/config_helper.exs index d75f9aaba7..31570bd053 100644 --- a/config/config_helper.exs +++ b/config/config_helper.exs @@ -21,6 +21,7 @@ defmodule ConfigHelper do :filecoin -> base_repos ++ [Explorer.Repo.Filecoin] :stability -> base_repos ++ [Explorer.Repo.Stability] :zksync -> base_repos ++ [Explorer.Repo.ZkSync] + :celo -> base_repos ++ [Explorer.Repo.Celo] :arbitrum -> base_repos ++ [Explorer.Repo.Arbitrum] _ -> base_repos end @@ -252,7 +253,7 @@ defmodule ConfigHelper do end @spec parse_json_env_var(String.t(), String.t()) :: any() - def parse_json_env_var(env_var, default_value) do + def parse_json_env_var(env_var, default_value \\ "{}") do env_var |> safe_get_env(default_value) |> Jason.decode!() @@ -292,7 +293,8 @@ defmodule ConfigHelper do "stability", "suave", "zetachain", - "zksync" + "zksync", + "celo" ] @spec chain_type() :: atom() | nil diff --git a/config/runtime.exs b/config/runtime.exs index db25f49215..22943edddc 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -451,6 +451,9 @@ config :explorer, Explorer.Chain.Cache.Uncles, ttl_check_interval: ConfigHelper.cache_ttl_check_interval(disable_indexer?), global_ttl: ConfigHelper.cache_global_ttl(disable_indexer?) +config :explorer, Explorer.Chain.Cache.CeloCoreContracts, + contracts: ConfigHelper.parse_json_env_var("CELO_CORE_CONTRACTS") + config :explorer, Explorer.ThirdPartyIntegrations.Sourcify, server_url: System.get_env("SOURCIFY_SERVER_URL") || "https://sourcify.dev/server", enabled: ConfigHelper.parse_bool_env_var("SOURCIFY_INTEGRATION_ENABLED"), @@ -1003,6 +1006,19 @@ config :indexer, Indexer.Fetcher.PolygonZkevm.TransactionBatch.Supervisor, ConfigHelper.chain_type() == :polygon_zkevm && ConfigHelper.parse_bool_env_var("INDEXER_POLYGON_ZKEVM_BATCHES_ENABLED") +config :indexer, Indexer.Fetcher.Celo.ValidatorGroupVotes, + batch_size: ConfigHelper.parse_integer_env_var("INDEXER_CELO_VALIDATOR_GROUP_VOTES_BATCH_SIZE", 200_000) + +celo_epoch_fetchers_enabled? = + ConfigHelper.chain_type() == :celo and + not ConfigHelper.parse_bool_env_var("INDEXER_DISABLE_CELO_EPOCH_FETCHER") + +config :indexer, Indexer.Fetcher.Celo.ValidatorGroupVotes.Supervisor, enabled: celo_epoch_fetchers_enabled? + +config :indexer, Indexer.Fetcher.Celo.EpochBlockOperations.Supervisor, + enabled: celo_epoch_fetchers_enabled?, + disabled?: not celo_epoch_fetchers_enabled? + Code.require_file("#{config_env()}.exs", "config/runtime") for config <- "../apps/*/config/runtime/#{config_env()}.exs" |> Path.expand(__DIR__) |> Path.wildcard() do diff --git a/config/runtime/dev.exs b/config/runtime/dev.exs index 2c831a5f36..b4ce37657a 100644 --- a/config/runtime/dev.exs +++ b/config/runtime/dev.exs @@ -126,6 +126,15 @@ config :explorer, Explorer.Repo.ZkSync, # separating repos for different CHAIN_TYPE is implemented only for the sake of keeping DB schema update relevant to the current chain type pool_size: 1 +# Configure Celo database +config :explorer, Explorer.Repo.Celo, + database: database, + hostname: hostname, + url: System.get_env("DATABASE_URL"), + # actually this repo is not started, and its pool size remains unused. + # separating repos for different CHAIN_TYPE is implemented only for the sake of keeping DB schema update relevant to the current chain type + pool_size: 1 + # Configure Rootstock database config :explorer, Explorer.Repo.RSK, database: database, diff --git a/config/runtime/prod.exs b/config/runtime/prod.exs index b88b354719..6f240fee48 100644 --- a/config/runtime/prod.exs +++ b/config/runtime/prod.exs @@ -96,6 +96,14 @@ config :explorer, Explorer.Repo.ZkSync, pool_size: 1, ssl: ExplorerConfigHelper.ssl_enabled?() +# Configures Celo database +config :explorer, Explorer.Repo.Celo, + url: System.get_env("DATABASE_URL"), + # actually this repo is not started, and its pool size remains unused. + # separating repos for different CHAIN_TYPE is implemented only for the sake of keeping DB schema update relevant to the current chain type + pool_size: 1, + ssl: ExplorerConfigHelper.ssl_enabled?() + # Configures Rootstock database config :explorer, Explorer.Repo.RSK, url: System.get_env("DATABASE_URL"), diff --git a/cspell.json b/cspell.json index 91a90a03e6..d6dee164e4 100644 --- a/cspell.json +++ b/cspell.json @@ -19,6 +19,7 @@ "AIRTABLE", "Aiubo", "alloc", + "alfajores", "amzootyukbugmx", "anytrust", "apikey", @@ -83,6 +84,7 @@ "CBOR", "Celestia", "cellspacing", + "celo", "certifi", "cfasync", "chainid", @@ -168,6 +170,7 @@ "enetunreach", "enoent", "epns", + "epochrewards", "Erigon", "errora", "errorb", @@ -192,9 +195,10 @@ "falala", "feelin", "FEVM", - "filecoin", "Filecoin", "Filesize", + "filecoin", + "fixidity", "fkey", "Floki", "fontawesome", @@ -221,6 +225,7 @@ "gettxinfo", "gettxreceiptstatus", "giga", + "goldtoken", "gqz", "granitegrey", "graphiql", @@ -284,6 +289,7 @@ "listcontracts", "lkve", "llhauc", + "lockedgold", "loggable", "LPAD", "LUKSO", @@ -428,6 +434,7 @@ "redix", "refetched", "regclass", + "registryproxy", "REINDEX", "relname", "relpages", @@ -476,6 +483,7 @@ "sourcify", "splitted", "srcset", + "stabletoken", "staker", "stakers", "stateroot", diff --git a/docker-compose/envs/common-blockscout.env b/docker-compose/envs/common-blockscout.env index 2b23e63402..205bf8407e 100644 --- a/docker-compose/envs/common-blockscout.env +++ b/docker-compose/envs/common-blockscout.env @@ -248,6 +248,9 @@ INDEXER_DISABLE_INTERNAL_TRANSACTIONS_FETCHER=false # INDEXER_ARBITRUM_BRIDGE_MESSAGES_TRACKING_ENABLED= # INDEXER_ARBITRUM_TRACKING_MESSAGES_ON_L1_RECHECK_INTERVAL= # INDEXER_ARBITRUM_MISSED_MESSAGES_RECHECK_INTERVAL= +# CELO_CORE_CONTRACTS= +# INDEXER_CELO_VALIDATOR_GROUP_VOTES_BATCH_SIZE=200000 +# INDEXER_DISABLE_CELO_EPOCH_FETCHER=false # INDEXER_ARBITRUM_MISSED_MESSAGES_BLOCKS_DEPTH= # INDEXER_REALTIME_FETCHER_MAX_GAP= # INDEXER_FETCHER_INIT_QUERY_LIMIT= @@ -410,4 +413,4 @@ TENDERLY_CHAIN_PATH= # SANITIZE_INCORRECT_WETH_CONCURRENCY=1 # PUBLIC_METRICS_ENABLED= # PUBLIC_METRICS_UPDATE_PERIOD_HOURS= -# CSV_EXPORT_LIMIT= \ No newline at end of file +# CSV_EXPORT_LIMIT=