Merge branch 'master' into show-unit-price-with-balance

README.md

@@ -1,6 +1,17 @@
-# BlockScout - Blockchain Explorer for EVM Chains :fire: 
+<p align="center">
+  <a href="https://blockscout.com">
+    <img width="200" src="https://blockscout.com/eth/mainnet/android-chrome-192x192.png" \>
+  </a>
+</p>
+
+<h1 align="center">BlockScout</h1>
+<p align="center">Blockchain Explorer for inspecting and analyzing EVM Chains.</p>
+<div align="center">
+  
 [![CircleCI](https://circleci.com/gh/poanetwork/blockscout.svg?style=svg&circle-token=f8823a3d0090407c11f87028c73015a331dbf604)](https://circleci.com/gh/poanetwork/blockscout) [![Coverage Status](https://coveralls.io/repos/github/poanetwork/blockscout/badge.svg?branch=master)](https://coveralls.io/github/poanetwork/blockscout?branch=master) [![Join the chat at https://gitter.im/poanetwork/blockscout](https://badges.gitter.im/poanetwork/blockscout.svg)](https://gitter.im/poanetwork/blockscout?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
+</div>
+
 BlockScout provides a comprehensive, easy-to-use interface for users to view, confirm, and inspect transactions on **all EVM** (Ethereum Virtual Machine) blockchains. This includes the Ethereum main and test networks as well as **Ethereum forks and sidechains**.
 
 Following is an overview of the project and instructions for [getting started](#getting-started).
@@ -96,6 +107,9 @@ _Additional runtime options:_
 
 ![BlockScout Example](explorer_example.gif)
 
+### Projects Utilizing BlockScout
+* [Oasis Labs](https://blockexplorer.oasiscloud.io/)
+
 ### Configuring Ethereum Classic and other EVM Chains
 **Note: Most of these modifications will be consolidated into a single file in the future.**
 

UPGRADING.md

@@ -0,0 +1,20 @@
+# Upgrading Guide
+
+### Migration scripts
+
+There is in the project a `scripts` folder that contains `SQL` files responsible to migrate data from the database.
+
+This script should be used if you already have an indexed database with a large amount of data.
+
+#### `address_current_token_balances_in_batches.sql`
+
+Is responsible to populate a new table using the `token_balances` table information.
+
+#### `internal_transaction_update_in_batches.sql`
+
+Is responsible to migrate data from the `transactions` table to the `internal_transactions` one in order to improve the application listing performance;
+
+#### `transaction_update_in_baches.sql`
+
+Parity call traces contain the input, but it was not put in the internal_transactions_params.
+Enforce input and call_type being non-NULL for calls in new constraints on internal_transactions.

apps/block_scout_web/assets/css/theme/_goerli_variables.scss

@@ -0,0 +1,3 @@
+$primary:       #20201A;
+$secondary:     #F0D96B;
+$tertiary:      #4A443A;

apps/explorer/lib/mix/tasks/migrate.transaction.info.ex

@@ -1,32 +0,0 @@
-defmodule Mix.Tasks.Migrate.Transaction.Info do
-  use Mix.Task
-
-  alias Ecto.Adapters.SQL
-  alias Explorer.Repo
-
-  @shortdoc "Migrates transaction info to internal transaction"
-
-  @moduledoc """
-  This task is reponsible to populate the `transaction_index` and
-  `block_number` at the `internal_transactions` table, using the
-  `transactions` info.
-  """
-
-  def run(_args) do
-    {:ok, _} = Application.ensure_all_started(:explorer)
-
-    SQL.query(
-      Repo,
-      """
-        UPDATE internal_transactions
-        SET
-          block_number = transactions.block_number,
-          transaction_index = transactions.index
-        FROM transactions
-        WHERE internal_transactions.transaction_hash = transactions.hash;
-      """,
-      [],
-      timeout: :infinity
-    )
-  end
-end

apps/explorer/priv/repo/migrations/scripts/address_current_token_balances_in_batches.sql

@@ -0,0 +1,69 @@
+DO $$
+DECLARE
+   row_count integer := 1;
+   batch_size  integer := 50000; -- HOW MANY ITEMS WILL BE UPDATED AT TIME
+   iterator  integer := batch_size;
+   affected integer;
+BEGIN
+  DROP TABLE IF EXISTS current_token_balance_temp;
+  -- CREATES TEMP TABLE TO STORE TOKEN BALANCES TO BE UPDATED
+  CREATE TEMP TABLE current_token_balance_temp(
+    address_hash bytea NOT NULL,
+    block_number bigint NOT NULL,
+    token_contract_address_hash bytea NOT NULL,
+    value numeric,
+    value_fetched_at timestamp without time zone,
+    inserted_at timestamp without time zone NOT NULL,
+    updated_at timestamp without time zone NOT NULL,
+    row_number integer
+  );
+  INSERT INTO current_token_balance_temp
+  SELECT DISTINCT ON (address_hash, token_contract_address_hash)
+    address_hash,
+    block_number,
+    token_contract_address_hash,
+    value,
+    value_fetched_at,
+    inserted_at,
+    updated_at,
+    ROW_NUMBER () OVER ()
+  FROM address_token_balances
+  WHERE value IS NOT NULL
+  ORDER BY address_hash, token_contract_address_hash, block_number DESC;
+
+  row_count := (SELECT count(*) FROM current_token_balance_temp);
+  RAISE NOTICE '% items to be updated', row_count;
+
+  -- ITERATES THROUGH THE ITEMS UNTIL THE TEMP TABLE IS EMPTY
+  WHILE row_count > 0 LOOP
+    -- INSERT THE TOKEN BALANCES AND RETURNS THE ADDRESS HASH AND TOKEN HASH TO BE DELETED
+    WITH updated_address_current_token_balances AS (
+      INSERT INTO address_current_token_balances (address_hash, block_number, token_contract_address_hash, value, value_fetched_at, inserted_at, updated_at)
+      SELECT
+        address_hash,
+        block_number,
+        token_contract_address_hash,
+        value,
+        value_fetched_at,
+        inserted_at,
+        updated_at
+      FROM current_token_balance_temp
+      WHERE current_token_balance_temp.row_number <= iterator
+      RETURNING address_hash, token_contract_address_hash
+    )
+    DELETE FROM current_token_balance_temp
+    WHERE (address_hash, token_contract_address_hash) IN (select address_hash, token_contract_address_hash from updated_address_current_token_balances);
+
+    GET DIAGNOSTICS affected = ROW_COUNT;
+    RAISE NOTICE '-> % address current token balances updated!', affected;
+
+    -- COMMITS THE BATCH UPDATES
+    CHECKPOINT;
+
+    -- UPDATES THE COUNTER SO IT DOESN'T TURN INTO AN INFINITE LOOP
+    row_count := (SELECT count(*) FROM current_token_balance_temp);
+    iterator := iterator + batch_size;
+    RAISE NOTICE '-> % counter', row_count;
+    RAISE NOTICE '-> % next batch', iterator;
+  END LOOP;
+END $$;

apps/explorer/priv/repo/migrations/scripts/internal_transaction_update_in_batches.sql

@@ -0,0 +1,59 @@
+DO $$
+DECLARE
+   row_count integer := 1;
+   batch_size  integer := 50000; -- HOW MANY ITEMS WILL BE UPDATED AT TIME
+   iterator  integer := batch_size;
+   affected integer;
+BEGIN
+  DROP TABLE IF EXISTS transactions_temp;
+
+  -- CREATES TEMP TABLE TO STORE TRANSACTIONS TO BE UPDATED
+  CREATE TEMP TABLE transactions_temp(hash bytea, block_number integer, index integer, row_number integer);
+  INSERT INTO transactions_temp
+  SELECT
+    t.hash,
+    t.block_number,
+    t.index,
+    ROW_NUMBER () OVER ()
+  FROM transactions t
+  INNER JOIN internal_transactions it ON t.hash = it.transaction_hash AND it.block_number IS NULL
+  WHERE
+    t.hash = it.transaction_hash
+    AND it.block_number IS NULL
+  ;
+
+  row_count := (SELECT count(*) FROM transactions_temp);
+
+  RAISE NOTICE '% items to be updated', row_count;
+
+  -- ITERATES THROUGH THE ITEMS UNTIL THE TEMP TABLE IS EMPTY
+  WHILE row_count > 0 LOOP
+    -- UPDATES INTERNAL TRANSACTION AND RETURNS THE HASH TO BE DELETED
+    WITH updated_internal_transactions AS (
+      UPDATE internal_transactions
+      SET
+        block_number = transactions_temp.block_number,
+        transaction_index = transactions_temp.index
+      FROM transactions_temp
+      WHERE internal_transactions.transaction_hash = transactions_temp.hash
+      AND transactions_temp.row_number <= iterator
+      RETURNING transactions_temp.hash
+    )
+    -- DELETES THE ITENS UPDATED FROM THE TEMP TABLE
+    DELETE FROM transactions_temp tt
+    USING  updated_internal_transactions uit
+    WHERE  tt.hash = uit.hash;
+
+    GET DIAGNOSTICS affected = ROW_COUNT;
+    RAISE NOTICE '-> % internal transactions updated!', affected;
+
+    CHECKPOINT; -- COMMITS THE BATCH UPDATES
+
+    -- UPDATES THE COUNTER SO IT DOESN'T TURN INTO AN INFINITE LOOP
+    row_count := (SELECT count(*) FROM transactions_temp);
+    iterator := iterator + batch_size;
+
+    RAISE NOTICE '-> % counter', row_count;
+    RAISE NOTICE '-> % next batch', iterator;
+  END LOOP;
+END $$;

apps/explorer/priv/repo/migrations/scripts/transaction_update_in_batches.sql

@@ -0,0 +1,55 @@
+DO $$
+DECLARE
+   row_count integer := 1;
+   batch_size  integer := 50000; -- HOW MANY ITEMS WILL BE UPDATED AT TIME
+   iterator  integer := batch_size;
+   affected integer;
+BEGIN
+  DROP TABLE IF EXISTS transactions_error_itx_indexed_at_temp;
+
+  -- CREATES TEMP TABLE TO STORE TRANSACTIONS TO BE UPDATED
+  CREATE TEMP TABLE transactions_error_itx_indexed_at_temp(hash bytea, row_number integer);
+
+  INSERT INTO transactions_error_itx_indexed_at_temp
+  SELECT
+    DISTINCT it.transaction_hash,
+    ROW_NUMBER () OVER ()
+  FROM internal_transactions it
+  WHERE it.type = 'call'
+  AND it.input IS NULL;
+
+  row_count := (SELECT count(*) FROM transactions_error_itx_indexed_at_temp);
+
+  RAISE NOTICE '% items to be updated', row_count;
+
+  -- ITERATES THROUGH THE ITEMS UNTIL THE TEMP TABLE IS EMPTY
+  WHILE row_count > 0 LOOP
+    -- UPDATES TRANSACTION AND RETURNS THE HASH TO BE DELETED
+    WITH updated_transactions AS (
+      UPDATE transactions
+      SET
+        internal_transactions_indexed_at = null,
+        error = null
+      FROM transactions_error_itx_indexed_at_temp
+      WHERE transactions.hash = transactions_error_itx_indexed_at_temp.hash
+      AND transactions_error_itx_indexed_at_temp.row_number <= iterator
+      RETURNING transactions_error_itx_indexed_at_temp.hash
+    )
+    -- DELETES THE ITENS UPDATED FROM THE TEMP TABLE
+    DELETE FROM transactions_error_itx_indexed_at_temp tt
+    USING  updated_transactions uit
+    WHERE  tt.hash = uit.hash;
+
+    GET DIAGNOSTICS affected = ROW_COUNT;
+    RAISE NOTICE '-> % transactions updated!', affected;
+
+    CHECKPOINT; -- COMMITS THE BATCH UPDATES
+
+    -- UPDATES THE COUNTER SO IT DOESN'T TURN INTO AN INFINITE LOOP
+    row_count := (SELECT count(*) FROM transactions_error_itx_indexed_at_temp);
+    iterator := iterator + batch_size;
+
+    RAISE NOTICE '-> % counter', row_count;
+    RAISE NOTICE '-> % next batch', iterator;
+  END LOOP;
+END $$;

apps/indexer/lib/indexer/token_balances.ex

@@ -9,6 +9,9 @@ defmodule Indexer.TokenBalances do
   alias Explorer.Token.BalanceReader
   alias Indexer.TokenBalance
 
+  # The timeout used for each process opened by Task.async_stream/3. Default 15s.
+  @task_timeout 15000
+
   @doc """
   Fetches TokenBalances from specific Addresses and Blocks in the Blockchain
 
@@ -26,12 +29,14 @@ defmodule Indexer.TokenBalances do
   """
   def fetch_token_balances_from_blockchain([]), do: {:ok, []}
 
-  def fetch_token_balances_from_blockchain(token_balances) do
+  def fetch_token_balances_from_blockchain(token_balances, opts \\ []) do
     Logger.debug(fn -> "fetching #{Enum.count(token_balances)} token balances" end)
 
+    task_timeout = Keyword.get(opts, :timeout, @task_timeout)
+
     requested_token_balances =
       token_balances
-      |> Task.async_stream(&fetch_token_balance/1, on_timeout: :kill_task)
+      |> Task.async_stream(&fetch_token_balance/1, timeout: task_timeout, on_timeout: :kill_task)
       |> Stream.map(&format_task_results/1)
       |> Enum.filter(&ignore_killed_task/1)
 

apps/indexer/test/indexer/token_balances_test.exs

@@ -84,9 +84,9 @@ defmodule Indexer.TokenBalancesTest do
       ]
 
       get_balance_from_blockchain()
-      get_balance_from_blockchain_with_timeout()
+      get_balance_from_blockchain_with_timeout(200)
 
-      {:ok, result} = TokenBalances.fetch_token_balances_from_blockchain(token_balance_params)
+      {:ok, result} = TokenBalances.fetch_token_balances_from_blockchain(token_balance_params, timeout: 100)
 
       assert length(result) == 1
     end
@@ -190,12 +190,21 @@ defmodule Indexer.TokenBalancesTest do
     )
   end
 
-  defp get_balance_from_blockchain_with_timeout() do
+  defp get_balance_from_blockchain_with_timeout(timeout) do
     expect(
       EthereumJSONRPC.Mox,
       :json_rpc,
       fn [%{id: _, method: _, params: [%{data: _, to: _}, _]}], _options ->
-        :timer.sleep(5001)
+        :timer.sleep(timeout)
+
+        {:ok,
+         [
+           %{
+             id: "balanceOf",
+             jsonrpc: "2.0",
+             result: "0x00000000000000000000000000000000000000000000d3c21bcecceda1000000"
+           }
+         ]}
       end
     )
   end