From 0423c18d3a0dfda318621cc8c353ee895e728045 Mon Sep 17 00:00:00 2001 From: nikitosing <32202610+nikitosing@users.noreply.github.com> Date: Wed, 30 Oct 2024 16:36:53 +0300 Subject: [PATCH] fix: Fix contract codes fetching for zksync chain type (#11055) * fix: Fix contract codes fetching for zksync chain type * Process review comments * Fix tests * Fix tests * Fix tests --- .../views/address_contract_view.ex | 8 +- apps/explorer/config/config.exs | 1 + apps/explorer/config/runtime/test.exs | 1 + apps/explorer/lib/explorer/application.ex | 3 +- apps/explorer/lib/explorer/chain/address.ex | 13 +++ .../migrator/refetch_contract_codes.ex | 80 +++++++++++++++++++ ...1028102853_add_contract_code_refetched.exs | 13 +++ ...anitize_duplicated_log_index_logs_test.exs | 8 +- apps/explorer/test/support/factory.ex | 13 +++ .../lib/indexer/transform/addresses.ex | 17 ++-- config/runtime.exs | 8 ++ docker-compose/envs/common-blockscout.env | 4 + 12 files changed, 155 insertions(+), 14 deletions(-) create mode 100644 apps/explorer/lib/explorer/migrator/refetch_contract_codes.ex create mode 100644 apps/explorer/priv/zk_sync/migrations/20241028102853_add_contract_code_refetched.exs diff --git a/apps/block_scout_web/lib/block_scout_web/views/address_contract_view.ex b/apps/block_scout_web/lib/block_scout_web/views/address_contract_view.ex index efb3dd168f..0b91ef2cd8 100644 --- a/apps/block_scout_web/lib/block_scout_web/views/address_contract_view.ex +++ b/apps/block_scout_web/lib/block_scout_web/views/address_contract_view.ex @@ -118,14 +118,14 @@ defmodule BlockScoutWeb.AddressContractView do {:ok, contract_code} end - def creation_code(%Address{contracts_creation_internal_transaction: %InternalTransaction{}} = address) do - address.contracts_creation_internal_transaction.init - end - def creation_code(%Address{contracts_creation_transaction: %Transaction{}} = address) do address.contracts_creation_transaction.input end + def creation_code(%Address{contracts_creation_internal_transaction: %InternalTransaction{}} = address) do + address.contracts_creation_internal_transaction.init + end + def creation_code(%Address{contracts_creation_transaction: nil}) do nil end diff --git a/apps/explorer/config/config.exs b/apps/explorer/config/config.exs index 681086d350..a1977ac738 100644 --- a/apps/explorer/config/config.exs +++ b/apps/explorer/config/config.exs @@ -132,6 +132,7 @@ config :explorer, Explorer.Migrator.SanitizeMissingTokenBalances, enabled: true config :explorer, Explorer.Migrator.SanitizeReplacedTransactions, enabled: true config :explorer, Explorer.Migrator.ReindexInternalTransactionsWithIncompatibleStatus, enabled: true config :explorer, Explorer.Migrator.SanitizeDuplicatedLogIndexLogs, enabled: true +config :explorer, Explorer.Migrator.RefetchContractCodes, enabled: true config :explorer, Explorer.Chain.Fetcher.CheckBytecodeMatchingOnDemand, enabled: true diff --git a/apps/explorer/config/runtime/test.exs b/apps/explorer/config/runtime/test.exs index 177cfd6b2b..f4be94ed72 100644 --- a/apps/explorer/config/runtime/test.exs +++ b/apps/explorer/config/runtime/test.exs @@ -53,6 +53,7 @@ config :explorer, Explorer.Migrator.SanitizeMissingTokenBalances, enabled: false config :explorer, Explorer.Migrator.SanitizeReplacedTransactions, enabled: false config :explorer, Explorer.Migrator.ReindexInternalTransactionsWithIncompatibleStatus, enabled: false config :explorer, Explorer.Migrator.SanitizeDuplicatedLogIndexLogs, enabled: false +config :explorer, Explorer.Migrator.RefetchContractCodes, enabled: false config :explorer, realtime_events_sender: Explorer.Chain.Events.SimpleSender diff --git a/apps/explorer/lib/explorer/application.ex b/apps/explorer/lib/explorer/application.ex index 8e835b6528..33aaed8792 100644 --- a/apps/explorer/lib/explorer/application.ex +++ b/apps/explorer/lib/explorer/application.ex @@ -155,7 +155,8 @@ defmodule Explorer.Application do ]), configure_mode_dependent_process(Explorer.Migrator.SanitizeMissingTokenBalances, :indexer), configure_mode_dependent_process(Explorer.Migrator.SanitizeReplacedTransactions, :indexer), - configure_mode_dependent_process(Explorer.Migrator.ReindexInternalTransactionsWithIncompatibleStatus, :indexer) + configure_mode_dependent_process(Explorer.Migrator.ReindexInternalTransactionsWithIncompatibleStatus, :indexer), + Explorer.Migrator.RefetchContractCodes |> configure() |> configure_chain_type_dependent_process(:zksync) ] |> List.flatten() diff --git a/apps/explorer/lib/explorer/chain/address.ex b/apps/explorer/lib/explorer/chain/address.ex index f2c4a4ed2a..1c86455392 100644 --- a/apps/explorer/lib/explorer/chain/address.ex +++ b/apps/explorer/lib/explorer/chain/address.ex @@ -58,6 +58,13 @@ defmodule Explorer.Chain.Address.Schema do ] end + :zksync -> + quote do + [ + field(:contract_code_refetched, :boolean) + ] + end + _ -> [] end) @@ -142,6 +149,9 @@ defmodule Explorer.Chain.Address do :filecoin -> ~w(filecoin_id filecoin_robust filecoin_actor_type)a + :zksync -> + ~w(contract_code_refetched)a + _ -> [] end) @@ -196,6 +206,9 @@ defmodule Explorer.Chain.Address do * `filecoin_id_address` - short f0 Filecoin address that may change during chain reorgs * `filecoin_actor_type` - type of actor associated with the Filecoin address """ + :zksync -> """ + * `contract_code_refetched` - true when Explorer.Migrator.RefetchContractCodes handled this address, or it's unnecessary (for addresses inserted after this) + """ _ -> "" end} `fetched_coin_balance` and `fetched_coin_balance_block_number` may be updated when a new coin_balance row is fetched. diff --git a/apps/explorer/lib/explorer/migrator/refetch_contract_codes.ex b/apps/explorer/lib/explorer/migrator/refetch_contract_codes.ex new file mode 100644 index 0000000000..a20574417c --- /dev/null +++ b/apps/explorer/lib/explorer/migrator/refetch_contract_codes.ex @@ -0,0 +1,80 @@ +defmodule Explorer.Migrator.RefetchContractCodes do + @moduledoc """ + Refetch contract_code for. Migration created for running on zksync chain type. + It has an issue with created contract code derived from internal transactions. Such codes are not correct. + So, this migration fetches for all current smart contracts actual bytecode from the JSON RPC node. + """ + + use Explorer.Migrator.FillingMigration + + import Ecto.Query + + alias Explorer.Chain.{Address, Data, Import} + alias Explorer.Chain.Hash.Address, as: AddressHash + alias Explorer.Chain.Import.Runner.Addresses + alias Explorer.Migrator.FillingMigration + alias Explorer.Repo + + require Logger + + @migration_name "refetch_contract_codes" + + @impl FillingMigration + def migration_name, do: @migration_name + + @impl FillingMigration + def last_unprocessed_identifiers(state) do + limit = batch_size() * concurrency() + + ids = + unprocessed_data_query() + |> select([address], address.hash) + |> limit(^limit) + |> Repo.all(timeout: :infinity) + + {ids, state} + end + + @impl FillingMigration + def unprocessed_data_query do + Address + |> where([address], not is_nil(address.contract_code) and not address.contract_code_refetched) + end + + @impl FillingMigration + def update_batch(address_hashes) do + json_rpc_named_arguments = Application.get_env(:explorer, :json_rpc_named_arguments) + + address_hashes + |> Enum.map(&address_to_fetch_code_params/1) + |> EthereumJSONRPC.fetch_codes(json_rpc_named_arguments) + |> case do + {:ok, create_address_codes} -> + addresses_params = create_address_codes.params_list |> Enum.map(¶m_to_address/1) |> Enum.sort_by(& &1.hash) + + Addresses.insert(Repo, addresses_params, %{ + timeout: :infinity, + on_conflict: {:replace, [:contract_code, :contract_code_refetched, :updated_at]}, + timestamps: Import.timestamps() + }) + + {:error, reason} -> + Logger.error(fn -> ["failed to fetch contract codes: ", inspect(reason)] end, + error_count: Enum.count(address_hashes) + ) + end + end + + @impl FillingMigration + def update_cache, do: :ok + + defp address_to_fetch_code_params(address_hash) do + %{block_quantity: "latest", address: to_string(address_hash)} + end + + defp param_to_address(%{code: bytecode, address: address_hash}) do + {:ok, address_hash} = AddressHash.cast(address_hash) + {:ok, bytecode} = Data.cast(bytecode) + %{hash: address_hash, contract_code: bytecode, contract_code_refetched: true} + end +end diff --git a/apps/explorer/priv/zk_sync/migrations/20241028102853_add_contract_code_refetched.exs b/apps/explorer/priv/zk_sync/migrations/20241028102853_add_contract_code_refetched.exs new file mode 100644 index 0000000000..7f24a305d1 --- /dev/null +++ b/apps/explorer/priv/zk_sync/migrations/20241028102853_add_contract_code_refetched.exs @@ -0,0 +1,13 @@ +defmodule Explorer.Repo.ZkSync.Migrations.AddContractCodeRefetched do + use Ecto.Migration + + def change do + alter table(:addresses) do + add(:contract_code_refetched, :boolean, default: false) + end + + execute(""" + ALTER TABLE addresses ALTER COLUMN contract_code_refetched SET DEFAULT true; + """) + end +end diff --git a/apps/explorer/test/explorer/migrator/sanitize_duplicated_log_index_logs_test.exs b/apps/explorer/test/explorer/migrator/sanitize_duplicated_log_index_logs_test.exs index c9973ef618..7d0c5cfb23 100644 --- a/apps/explorer/test/explorer/migrator/sanitize_duplicated_log_index_logs_test.exs +++ b/apps/explorer/test/explorer/migrator/sanitize_duplicated_log_index_logs_test.exs @@ -7,7 +7,7 @@ defmodule Explorer.Migrator.SanitizeDuplicatedLogIndexLogsTest do alias Explorer.Chain.Token.Instance alias Explorer.Migrator.{SanitizeDuplicatedLogIndexLogs, MigrationStatus} - if Application.compile_env(:explorer, :chain_type) != :celo do + if Application.compile_env(:explorer, :chain_type) in [:polygon_zkevm, :rsk, :filecoin] do describe "Sanitize duplicated log index logs" do test "correctly identifies and updates duplicated log index logs" do block = insert(:block) @@ -24,7 +24,7 @@ defmodule Explorer.Migrator.SanitizeDuplicatedLogIndexLogsTest do assert MigrationStatus.get_status("sanitize_duplicated_log_index_logs") == nil SanitizeDuplicatedLogIndexLogs.start_link([]) - Process.sleep(300) + :timer.sleep(500) assert MigrationStatus.get_status("sanitize_duplicated_log_index_logs") == "completed" assert BackgroundMigrations.get_sanitize_duplicated_log_index_logs_finished() == true @@ -103,7 +103,7 @@ defmodule Explorer.Migrator.SanitizeDuplicatedLogIndexLogsTest do assert MigrationStatus.get_status("sanitize_duplicated_log_index_logs") == nil SanitizeDuplicatedLogIndexLogs.start_link([]) - Process.sleep(300) + :timer.sleep(500) assert MigrationStatus.get_status("sanitize_duplicated_log_index_logs") == "completed" assert BackgroundMigrations.get_sanitize_duplicated_log_index_logs_finished() == true @@ -138,7 +138,7 @@ defmodule Explorer.Migrator.SanitizeDuplicatedLogIndexLogsTest do assert MigrationStatus.get_status("sanitize_duplicated_log_index_logs") == nil SanitizeDuplicatedLogIndexLogs.start_link([]) - Process.sleep(100) + :timer.sleep(100) assert MigrationStatus.get_status("sanitize_duplicated_log_index_logs") == "completed" assert BackgroundMigrations.get_sanitize_duplicated_log_index_logs_finished() == true diff --git a/apps/explorer/test/support/factory.ex b/apps/explorer/test/support/factory.ex index c1390bfb35..3a78abcc6d 100644 --- a/apps/explorer/test/support/factory.ex +++ b/apps/explorer/test/support/factory.ex @@ -265,6 +265,19 @@ defmodule Explorer.Factory do %Address{ hash: address_hash() } + |> Map.merge(address_factory_chain_type_fields()) + end + + case Application.compile_env(:explorer, :chain_type) do + :zksync -> + defp address_factory_chain_type_fields() do + %{ + contract_code_refetched: true + } + end + + _ -> + defp address_factory_chain_type_fields(), do: %{} end def address_name_factory do diff --git a/apps/indexer/lib/indexer/transform/addresses.ex b/apps/indexer/lib/indexer/transform/addresses.ex index bec6ee4c73..59a9e0b837 100644 --- a/apps/indexer/lib/indexer/transform/addresses.ex +++ b/apps/indexer/lib/indexer/transform/addresses.ex @@ -73,11 +73,18 @@ defmodule Indexer.Transform.Addresses do %{from: :block_number, to: :fetched_coin_balance_block_number}, %{from: :to_address_hash, to: :hash} ], - [ - %{from: :block_number, to: :fetched_coin_balance_block_number}, - %{from: :created_contract_address_hash, to: :hash}, - %{from: :created_contract_code, to: :contract_code} - ] + if Application.compile_env(:explorer, :chain_type) == :zksync do + [ + %{from: :block_number, to: :fetched_coin_balance_block_number}, + %{from: :created_contract_address_hash, to: :hash} + ] + else + [ + %{from: :block_number, to: :fetched_coin_balance_block_number}, + %{from: :created_contract_address_hash, to: :hash}, + %{from: :created_contract_code, to: :contract_code} + ] + end ], codes: [ [ diff --git a/config/runtime.exs b/config/runtime.exs index 1ff0ad3901..a675e1d333 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -619,6 +619,14 @@ config :explorer, Explorer.Migrator.RestoreOmittedWETHTransfers, batch_size: ConfigHelper.parse_integer_env_var("MIGRATION_RESTORE_OMITTED_WETH_TOKEN_TRANSFERS_BATCH_SIZE", 50), timeout: ConfigHelper.parse_time_env_var("MIGRATION_RESTORE_OMITTED_WETH_TOKEN_TRANSFERS_TIMEOUT", "250ms") +config :explorer, Explorer.Migrator.SanitizeDuplicatedLogIndexLogs, + concurrency: ConfigHelper.parse_integer_env_var("MIGRATION_SANITIZE_DUPLICATED_LOG_INDEX_LOGS_CONCURRENCY", 10), + batch_size: ConfigHelper.parse_integer_env_var("MIGRATION_SANITIZE_DUPLICATED_LOG_INDEX_LOGS_BATCH_SIZE", 500) + +config :explorer, Explorer.Migrator.RefetchContractCodes, + concurrency: ConfigHelper.parse_integer_env_var("MIGRATION_REFETCH_CONTRACT_CODES_CONCURRENCY", 5), + batch_size: ConfigHelper.parse_integer_env_var("MIGRATION_REFETCH_CONTRACT_CODES_BATCH_SIZE", 100) + config :explorer, Explorer.Migrator.ShrinkInternalTransactions, enabled: ConfigHelper.parse_bool_env_var("SHRINK_INTERNAL_TRANSACTIONS_ENABLED"), batch_size: ConfigHelper.parse_integer_env_var("SHRINK_INTERNAL_TRANSACTIONS_BATCH_SIZE", 100), diff --git a/docker-compose/envs/common-blockscout.env b/docker-compose/envs/common-blockscout.env index ef36f375f6..35872c4e75 100644 --- a/docker-compose/envs/common-blockscout.env +++ b/docker-compose/envs/common-blockscout.env @@ -430,6 +430,10 @@ EIP_1559_ELASTICITY_MULTIPLIER=2 # MIGRATION_RESTORE_OMITTED_WETH_TOKEN_TRANSFERS_CONCURRENCY= # MIGRATION_RESTORE_OMITTED_WETH_TOKEN_TRANSFERS_BATCH_SIZE= # MIGRATION_RESTORE_OMITTED_WETH_TOKEN_TRANSFERS_TIMEOUT= +# MIGRATION_SANITIZE_DUPLICATED_LOG_INDEX_LOGS_CONCURRENCY= +# MIGRATION_SANITIZE_DUPLICATED_LOG_INDEX_LOGS_BATCH_SIZE= +# MIGRATION_REFETCH_CONTRACT_CODES_BATCH_SIZE= +# MIGRATION_REFETCH_CONTRACT_CODES_CONCURRENCY= SOURCIFY_INTEGRATION_ENABLED=false SOURCIFY_SERVER_URL= SOURCIFY_REPO_URL=