diff --git a/apps/explorer/lib/explorer/chain/address.ex b/apps/explorer/lib/explorer/chain/address.ex index 845b17f981..d422faf7b7 100644 --- a/apps/explorer/lib/explorer/chain/address.ex +++ b/apps/explorer/lib/explorer/chain/address.ex @@ -22,7 +22,9 @@ defmodule Explorer.Chain.Address do * `fetched_coin_balance_block_number` - the `t:Explorer.Chain.Block.t/0` `t:Explorer.Chain.Block.block_number/0` for which `fetched_coin_balance` was fetched * `hash` - the hash of the address's public key - * `contract_code` - the code of the contract when an Address is a contract + * `contract_code` - the binary code of the contract when an Address is a contract. The human-readable + Solidity source code is in `smart_contract` `t:Explorer.Chain.SmartContract.t/0` `contract_source_code` *if* the + contract has been verified * `names` - names known for the address * `inserted_at` - when this address was inserted * `updated_at` when this address was last updated diff --git a/apps/explorer/lib/explorer/chain/smart_contract.ex b/apps/explorer/lib/explorer/chain/smart_contract.ex index 07d51b2216..7e78c7aa17 100644 --- a/apps/explorer/lib/explorer/chain/smart_contract.ex +++ b/apps/explorer/lib/explorer/chain/smart_contract.ex @@ -12,12 +12,189 @@ defmodule Explorer.Chain.SmartContract do use Explorer.Schema + @typedoc """ + The name of a parameter to a function or event. + """ + @type parameter_name :: String.t() + + @typedoc """ + Canonical Input or output [type](https://solidity.readthedocs.io/en/develop/abi-spec.html#types). + + * `"address"` - equivalent to `uint160`, except for the assumed interpretation and language typing. For computing the + function selector, `address` is used. + * `"bool"` - equivalent to uint8 restricted to the values 0 and 1. For computing the function selector, bool is used. + * `bytes`: dynamic sized byte sequence + * `"bytes"` - binary type of `M` bytes, `0 < M <= 32`. + * `"fixed"` - synonym for `"fixed128x18". For computing the function selection, `"fixed128x8"` has to be used. + * `"fixedx"` - signed fixed-point decimal number of `M` bits, `8 <= M <= 256`, `M % 8 ==0`, and `0 < N <= 80`, + which denotes the value `v` as `v / (10 ** N)`. + * `"function" - an address (`20` bytes) followed by a function selector (`4` bytes). Encoded identical to `bytes24`. + * `"int"` - synonym for `"int256"`. For computing the function selector `"int256"` has to be used. + * `"int"` - two’s complement signed integer type of `M` bits, `0 < M <= 256`, `M % 8 == 0`. + * `"string"` - dynamic sized unicode string assumed to be UTF-8 encoded. + * `"tuple"` - a tuple. + * `"(,,...,)"` - tuple consisting of the `t:type/0`s ``, …, `Tn`, `n >= 0`. + * `"[]"` - a variable-length array of elements of the given `type`. + * `"[M]"` - a fixed-length array of `M` elements, `M >= 0`, of the given `t:type/0`. + * `"ufixed"` - synonym for `"ufixed128x18". For computing the function selection, `"ufixed128x8"` has to be used. + * `"ufixedx"` - unsigned variant of `"fixedx"` + * `"uint"` - synonym for `"uint256"`. For computing the function selector `"uint256"` has to be used. + * `"uint"` - unsigned integer type of `M` bits, `0 < M <= 256`, `M % 8 == 0.` e.g. `uint32`, `uint8`, `uint256`. + """ + @type type :: String.t() + + @typedoc """ + Name of component. + """ + @type component_name :: String.t() + + @typedoc """ + A component of a [tuple](https://solidity.readthedocs.io/en/develop/abi-spec.html#handling-tuple-types). + + * `"name"` - name of the component. + * `"type"` - `t:type/0`. + """ + @type component :: %{String.t() => component_name() | type()} + + @typedoc """ + The components of a [tuple](https://solidity.readthedocs.io/en/develop/abi-spec.html#handling-tuple-types). + """ + @type components :: [component()] + + @typedoc """ + * `"event"` + """ + @type event_type :: String.t() + + @typedoc """ + Name of an event in an `t:abi/0`. + """ + @type event_name :: String.t() + + @typedoc """ + * `true` - if field is part of the `t:Explorer.Chain.Log.t/0` `topics`. + * `false` - if field is part of the `t:Explorer.Chain.Log.t/0` `data`. + """ + @type indexed :: boolean() + + @typedoc """ + * `"name"` - `t:parameter_name/0`. + * `"type"` - `t:type/0`. + * `"components" `- `t:components/0` used when `"type"` is a tuple type. + * `"indexed"` - `t:indexed/0`. + """ + @type event_input :: %{String.t() => parameter_name() | type() | components() | indexed()} + + @typedoc """ + * `true` - event was declared as `anonymous`. + * `false` - otherwise. + """ + @type anonymous :: boolean() + + @typedoc """ + * `"type" - `t:event_type/0` + * `"name"` - `t:event_name/0` + * `"inputs"` - `t:list/0` of `t:event_input/0`. + * `"anonymous"` - t:anonymous/0` + """ + @type event_description :: %{String.t() => term()} + + @typedoc """ + * `"function"` + * `"constructor"` + * `"fallback"` - the default, unnamed function + """ + @type function_type :: String.t() + + @typedoc """ + Name of a function in an `t:abi/0`. + """ + @type function_name :: String.t() + + @typedoc """ + * `"name"` - t:parameter_name/0`. + * `"type"` - `t:type/0`. + * `"components"` - `t:components/0` used when `"type"` is a tuple type. + """ + @type function_input :: %{String.t() => parameter_name() | type() | components()} + + @typedoc """ + * `"type" - `t:type/0` + """ + @type function_output :: %{String.t() => type()} + + @typedoc """ + * `"pure"` - [specified to not read blockchain state](https://solidity.readthedocs.io/en/develop/contracts.html#pure-functions). + * `"view"` - [specified to not modify the blockchain state](https://solidity.readthedocs.io/en/develop/contracts.html#view-functions). + * `"nonpayable"` - function does not accept Ether. + **NOTE**: Sending non-zero Ether to non-payable function will revert the transaction. + * `"payable"` - function accepts Ether. + """ + @type state_mutability :: String.t() + + @typedoc """ + **Deprecated:** Use `t:function_description/0` `"stateMutability"`: + + * `true` - `"payable"` + * `false` - `"pure"`, `"view"`, or `"nonpayable"`. + """ + @type payable :: boolean() + + @typedoc """ + **Deprecated:** Use `t:function_description/0` `"stateMutability"`: + + * `true` - `"pure"` or `"view"`. + * `false` - `"nonpayable"` or `"payable"`. + """ + @type constant :: boolean() + + @typedoc """ + The [function description](https://solidity.readthedocs.io/en/develop/abi-spec.html#json) for a function in the + `t:abi.t/0`. + + * `"type"` - `t:function_type/0` + * `"name" - `t:function_name/0` + * `"inputs` - `t:list/0` of `t:function_input/0`. + * `"outputs" - `t:list/0` of `t:output/0`. + * `"stateMutability"` - `t:state_mutability/0` + * `"payable"` - `t:payable/0`. + **WARNING:** Deprecated and will be removed in the future. Use `"stateMutability"` instead. + * `"constant"` - `t:constant/0`. + **WARNING:** Deprecated and will be removed in the future. Use `"stateMutability"` instead. + """ + @type function_description :: %{ + String.t() => + function_type() + | function_name() + | [function_input()] + | [function_output()] + | state_mutability() + | payable() + | constant() + } + + @typedoc """ + The [JSON ABI specification](https://solidity.readthedocs.io/en/develop/abi-spec.html#json) for a contract. + """ + @type abi :: [event_description | function_description] + + @typedoc """ + * `name` - the human-readable name of the smart contract. + * `compiler_version` - the version of the Solidity compiler used to compile `contract_source_code` with `optimization` + into `address` `t:Explorer.Chain.Address.t/0` `contract_code`. + * `optimization` - whether optimizations were turned on when compiling `contract_source_code` into `address` + `t:Explorer.Chain.Address.t/0` `contract_code`. + * `contract_source_code` - the Solidity source code that was compiled by `compiler_version` with `optimization` to + produce `address` `t:Explorer.Chain.Address.t/0` `contract_code`. + * `abi` - The [JSON ABI specification](https://solidity.readthedocs.io/en/develop/abi-spec.html#json) for this + contract. + """ @type t :: %Explorer.Chain.SmartContract{ name: String.t(), compiler_version: String.t(), optimization: boolean, contract_source_code: String.t(), - abi: {:array, :map} + abi: [function_description] } schema "smart_contracts" do diff --git a/apps/explorer/lib/explorer/chain/token_transfer.ex b/apps/explorer/lib/explorer/chain/token_transfer.ex index 5511d9eb80..6dd5a547d7 100644 --- a/apps/explorer/lib/explorer/chain/token_transfer.ex +++ b/apps/explorer/lib/explorer/chain/token_transfer.ex @@ -65,6 +65,8 @@ defmodule Explorer.Chain.TokenTransfer do @constant "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef" + @transfer_function_signature "0xa9059cbb" + @primary_key false schema "token_transfers" do field(:amount, :decimal) @@ -115,6 +117,11 @@ defmodule Explorer.Chain.TokenTransfer do """ def constant, do: @constant + @doc """ + ERC 20's transfer(address,uint256) function signature + """ + def transfer_function_signature, do: @transfer_function_signature + @spec fetch_token_transfers_from_token_hash(Hash.t(), [paging_options]) :: [] def fetch_token_transfers_from_token_hash(token_address_hash, options) do paging_options = Keyword.get(options, :paging_options, @default_paging_options) diff --git a/apps/explorer/lib/explorer/counters/average_block_time.ex b/apps/explorer/lib/explorer/counters/average_block_time.ex index 5bfce4082f..11e26e266b 100644 --- a/apps/explorer/lib/explorer/counters/average_block_time.ex +++ b/apps/explorer/lib/explorer/counters/average_block_time.ex @@ -64,13 +64,17 @@ defmodule Explorer.Counters.AverageBlockTime do # This is pretty naive, but we'll only ever be sorting 100 dates so I don't think # complex logic is really necessary here. - defp add_block(%{timestamps: timestamps} = state, block) do - timestamps = - [block | timestamps] - |> Enum.sort_by(fn {number, _} -> number end, &Kernel.>/2) - |> Enum.take(100) + defp add_block(%{timestamps: timestamps} = state, {new_number, _} = block) do + if Enum.any?(timestamps, fn {number, _} -> number == new_number end) do + state + else + timestamps = + [block | timestamps] + |> Enum.sort_by(fn {number, _} -> number end, &Kernel.>/2) + |> Enum.take(100) - %{state | timestamps: timestamps, average: average_distance(timestamps)} + %{state | timestamps: timestamps, average: average_distance(timestamps)} + end end defp average_distance([]), do: Duration.from_milliseconds(0) diff --git a/apps/explorer/priv/repo/migrations/scripts/update_address_current_token_balances_in_batches.sql b/apps/explorer/priv/repo/migrations/scripts/update_address_current_token_balances_in_batches.sql index ec3f1c5a16..61881df3bd 100644 --- a/apps/explorer/priv/repo/migrations/scripts/update_address_current_token_balances_in_batches.sql +++ b/apps/explorer/priv/repo/migrations/scripts/update_address_current_token_balances_in_batches.sql @@ -1,92 +1,125 @@ DO $$ DECLARE - row_count integer := 1; - batch_size integer := 50000; -- HOW MANY ITEMS WILL BE UPDATED AT TIME - iterator integer := batch_size; - affected integer; + total_count integer := 0; + completed_count integer := 0; + remaining_count integer := 0; + batch_size integer := 50000; -- HOW MANY ITEMS WILL BE UPDATED AT TIME + iterator integer := batch_size; + updated_count integer; + deleted_count integer; + start_time TIMESTAMP WITHOUT TIME ZONE := clock_timestamp(); + end_time TIMESTAMP WITHOUT TIME ZONE; + elapsed_time INTERVAL; + temp_start_time TIMESTAMP WITHOUT TIME ZONE; + temp_end_time TIMESTAMP WITHOUT TIME ZONE; + temp_elapsed_time INTERVAL; + update_start_time TIMESTAMP WITHOUT TIME ZONE; + update_end_time TIMESTAMP WITHOUT TIME ZONE; + update_elapsed_time INTERVAL; + per_row INTERVAL; BEGIN - DROP TABLE IF EXISTS address_token_temp; - CREATE TEMP TABLE address_token_temp + RAISE NOTICE 'Started at %', start_time; + + temp_start_time := clock_timestamp(); + + DROP TABLE IF EXISTS correct_address_current_token_block_numbers; + CREATE TEMP TABLE correct_address_current_token_block_numbers ( - address_hash bytea NOT NULL, - token_contract_address_hash bytea NOT NULL, + address_hash bytea NOT NULL, + token_contract_address_hash bytea NOT NULL, + block_number bigint NOT NULL, row_number integer ); - INSERT INTO address_token_temp - SELECT DISTINCT ON (address_hash, token_contract_address_hash) address_hash, - token_contract_address_hash, - ROW_NUMBER() OVER () + INSERT INTO correct_address_current_token_block_numbers + SELECT address_token_balances.address_hash, + address_token_balances.token_contract_address_hash, + MAX(address_token_balances.block_number), + ROW_NUMBER() OVER () FROM address_token_balances - WHERE value IS NOT NULL - ORDER BY address_hash, token_contract_address_hash; + INNER JOIN address_current_token_balances + ON address_current_token_balances.address_hash = + address_token_balances.address_hash AND + address_current_token_balances.token_contract_address_hash = + address_token_balances.token_contract_address_hash + GROUP BY address_token_balances.address_hash, + address_token_balances.token_contract_address_hash, + address_current_token_balances.block_number + HAVING MAX(address_token_balances.block_number) != address_current_token_balances.block_number; + + temp_end_time := clock_timestamp(); + temp_elapsed_time := temp_end_time - temp_start_time; + total_count := (SELECT COUNT(*) FROM correct_address_current_token_block_numbers); + + RAISE NOTICE 'correct_address_current_token_block_numbers TEMP table filled in %', temp_elapsed_time; - row_count := (SELECT COUNT(*) FROM address_token_temp); - RAISE NOTICE '% items to be updated', row_count; + remaining_count := total_count; + + RAISE NOTICE '% address_current_token_balances to be updated', remaining_count; + + update_start_time := clock_timestamp(); -- ITERATES THROUGH THE ITEMS UNTIL THE TEMP TABLE IS EMPTY - WHILE row_count > 0 + WHILE remaining_count > 0 LOOP UPDATE address_current_token_balances - SET block_number = new_address_current_token_balances.block_number, - value = new_address_current_token_balances.value, - inserted_at = new_address_current_token_balances.inserted_at, - updated_at = new_address_current_token_balances.updated_at - FROM ( - SELECT address_token_blocks.address_hash, - address_token_blocks.token_contract_address_hash, - address_token_blocks.block_number, - address_token_balances.value, - MIN(address_token_balances.inserted_at) OVER w AS inserted_at, - MAX(address_token_balances.updated_at) OVER w AS updated_at - FROM ( - SELECT address_token_batch.address_hash, - address_token_batch.token_contract_address_hash, - MAX(address_token_balances.block_number) AS block_number - FROM ( - SELECT address_hash, - token_contract_address_hash - FROM address_token_temp - WHERE address_token_temp.row_number <= iterator - ) AS address_token_batch - INNER JOIN address_token_balances - ON address_token_balances.address_hash = address_token_batch.address_hash AND - address_token_balances.token_contract_address_hash = - address_token_batch.token_contract_address_hash - GROUP BY address_token_batch.address_hash, - address_token_batch.token_contract_address_hash - ) AS address_token_blocks - INNER JOIN address_token_balances - ON address_token_balances.address_hash = address_token_blocks.address_hash AND - address_token_balances.token_contract_address_hash = - address_token_blocks.token_contract_address_hash AND - address_token_balances.block_number = address_token_blocks.block_number - WINDOW w AS (PARTITION BY address_token_balances.address_hash, address_token_balances.token_contract_address_hash) - ) AS new_address_current_token_balances - WHERE new_address_current_token_balances.address_hash = address_current_token_balances.address_hash + SET block_number = correct_address_current_token_block_numbers.block_number, + value = address_token_balances.value, + updated_at = NOW() + FROM correct_address_current_token_block_numbers, + address_token_balances + WHERE correct_address_current_token_block_numbers.row_number <= iterator + AND + correct_address_current_token_block_numbers.address_hash = address_current_token_balances.address_hash + AND + correct_address_current_token_block_numbers.token_contract_address_hash = + address_current_token_balances.token_contract_address_hash + AND + address_current_token_balances.block_number < correct_address_current_token_block_numbers.block_number AND - new_address_current_token_balances.token_contract_address_hash = + address_token_balances.address_hash = address_current_token_balances.address_hash + AND + address_token_balances.token_contract_address_hash = address_current_token_balances.token_contract_address_hash AND - (new_address_current_token_balances.block_number != address_current_token_balances.block_number OR - new_address_current_token_balances.value != address_current_token_balances.value); + address_token_balances.block_number = correct_address_current_token_block_numbers.block_number; - GET DIAGNOSTICS affected = ROW_COUNT; - RAISE NOTICE '-> % address current token balances updated.', affected; + GET DIAGNOSTICS updated_count = ROW_COUNT; + RAISE NOTICE '-> % address current token balances updated.', updated_count; DELETE - FROM address_token_temp - WHERE address_token_temp.row_number <= iterator; + FROM correct_address_current_token_block_numbers + WHERE correct_address_current_token_block_numbers.row_number <= iterator; - GET DIAGNOSTICS affected = ROW_COUNT; - RAISE NOTICE '-> % address tokens removed from queue.', affected; + GET DIAGNOSTICS deleted_count = ROW_COUNT; + RAISE NOTICE '-> % address tokens block numbers removed from queue.', deleted_count; -- COMMITS THE BATCH UPDATES CHECKPOINT; - -- UPDATES THE COUNTER SO IT DOESN'T TURN INTO AN INFINITE LOOP - row_count := (SELECT COUNT(*) FROM address_token_temp); + remaining_count := remaining_count - deleted_count; iterator := iterator + batch_size; - RAISE NOTICE '-> % counter', row_count; + RAISE NOTICE '-> % remaining', remaining_count; RAISE NOTICE '-> % next batch', iterator; + update_elapsed_time := clock_timestamp() - update_start_time; + completed_count := total_count - remaining_count; + per_row := update_elapsed_time / completed_count; + RAISE NOTICE '-> Estimated time until completion: %s', per_row * remaining_count; END LOOP; + + end_time := clock_timestamp(); + update_end_time := end_time; + update_elapsed_time = update_end_time - update_start_time; + + IF total_count > 0 THEN + per_row := update_elapsed_time / total_count; + ELSE + per_row := 0; + END IF; + + RAISE NOTICE 'address_current_token_balances updated in % (% per row)', update_elapsed_time, per_row; + + elapsed_time := end_time - start_time; + + RAISE NOTICE 'Ended at %s', end_time; + RAISE NOTICE 'Elapsed time: %', elapsed_time; END $$; diff --git a/apps/indexer/lib/indexer/block/realtime/fetcher.ex b/apps/indexer/lib/indexer/block/realtime/fetcher.ex index 36c9425ee5..9d8de8d0b3 100644 --- a/apps/indexer/lib/indexer/block/realtime/fetcher.ex +++ b/apps/indexer/lib/indexer/block/realtime/fetcher.ex @@ -12,18 +12,20 @@ defmodule Indexer.Block.Realtime.Fetcher do import EthereumJSONRPC, only: [integer_to_quantity: 1, quantity_to_integer: 1] import Indexer.Block.Fetcher, only: [async_import_tokens: 1, async_import_uncles: 1, fetch_and_import_range: 2] + alias ABI.TypeDecoder alias Ecto.Changeset alias EthereumJSONRPC.{FetchedBalances, Subscription} alias Explorer.Chain + alias Explorer.Chain.TokenTransfer + alias Explorer.Counters.AverageBlockTime alias Indexer.{AddressExtraction, Block, TokenBalances, Tracer} alias Indexer.Block.Realtime.{ConsensusEnsurer, TaskSupervisor} - - @polling_period 2_000 + alias Timex.Duration @behaviour Block.Fetcher @enforce_keys ~w(block_fetcher)a - defstruct ~w(block_fetcher subscription previous_number max_number_seen)a + defstruct ~w(block_fetcher subscription previous_number max_number_seen timer)a @type t :: %__MODULE__{ block_fetcher: %Block.Fetcher{ @@ -55,8 +57,13 @@ defmodule Indexer.Block.Realtime.Fetcher do def handle_continue({:init, subscribe_named_arguments}, %__MODULE__{subscription: nil} = state) when is_list(subscribe_named_arguments) do case EthereumJSONRPC.subscribe("newHeads", subscribe_named_arguments) do - {:ok, subscription} -> {:noreply, %__MODULE__{state | subscription: subscription}} - {:error, reason} -> {:stop, reason, state} + {:ok, subscription} -> + timer = schedule_polling() + + {:noreply, %__MODULE__{state | subscription: subscription, timer: timer}} + + {:error, reason} -> + {:stop, reason, state} end end @@ -67,25 +74,27 @@ defmodule Indexer.Block.Realtime.Fetcher do block_fetcher: %Block.Fetcher{} = block_fetcher, subscription: %Subscription{} = subscription, previous_number: previous_number, - max_number_seen: max_number_seen + max_number_seen: max_number_seen, + timer: timer } = state ) when is_binary(quantity) do number = quantity_to_integer(quantity) - # Subscriptions don't support getting all the blocks and transactions data, # so we need to go back and get the full block start_fetch_and_import(number, block_fetcher, previous_number, max_number_seen) new_max_number = new_max_number(number, max_number_seen) - schedule_polling() + :timer.cancel(timer) + new_timer = schedule_polling() {:noreply, %{ state | previous_number: number, - max_number_seen: new_max_number + max_number_seen: new_max_number, + timer: new_timer }} end @@ -100,22 +109,23 @@ defmodule Indexer.Block.Realtime.Fetcher do ) do {number, new_max_number} = case EthereumJSONRPC.fetch_block_number_by_tag("latest", json_rpc_named_arguments) do - {:ok, number} -> - start_fetch_and_import(number, block_fetcher, previous_number, max_number_seen) + {:ok, number} when is_nil(max_number_seen) or number > max_number_seen -> + start_fetch_and_import(number, block_fetcher, previous_number, number) - {number, new_max_number(number, max_number_seen)} + {max_number_seen, number} - {:error, _} -> + _ -> {previous_number, max_number_seen} end - schedule_polling() + timer = schedule_polling() {:noreply, %{ state | previous_number: number, - max_number_seen: new_max_number + max_number_seen: new_max_number, + timer: timer }} end @@ -124,7 +134,13 @@ defmodule Indexer.Block.Realtime.Fetcher do defp new_max_number(number, max_number_seen), do: max(number, max_number_seen) defp schedule_polling do - Process.send_after(self(), :poll_latest_block_number, @polling_period) + polling_period = + case AverageBlockTime.average_block_time() do + {:error, :disabled} -> 2_000 + block_time -> round(Duration.to_milliseconds(block_time) * 2) + end + + Process.send_after(self(), :poll_latest_block_number, polling_period) end @import_options ~w(address_hash_to_fetched_balance_block_number)a @@ -373,6 +389,31 @@ defmodule Indexer.Block.Realtime.Fetcher do end end + # 0xa9059cbb - signature of the transfer(address,uint256) function from the ERC-20 token specification. + # Although transaction input data can be faked we use this heuristics to filter simple token transfer internal transactions from indexing because they slow down realtime fetcher + defp fetch_internal_transactions?( + %{ + status: :ok, + created_contract_address_hash: nil, + input: unquote(TokenTransfer.transfer_function_signature()) <> params, + value: 0 + }, + _ + ) do + types = [:address, {:uint, 256}] + + try do + [_address, _value] = + params + |> Base.decode16!(case: :mixed) + |> TypeDecoder.decode_raw(types) + + false + rescue + _ -> true + end + end + # Input-less transactions are value-transfers only, so their internal transactions do not need to be indexed defp fetch_internal_transactions?(%{status: :ok, created_contract_address_hash: nil, input: "0x"}, _), do: false # Token transfers not transferred during contract creation don't need internal transactions as the token transfers