Merge branch 'master' into vb-block-rewards

CHANGELOG.md

@@ -1,12 +1,25 @@
 ## Current
 
 ### Features
+- [#2835](https://github.com/poanetwork/blockscout/pull/2835), [#2871](https://github.com/poanetwork/blockscout/pull/2871), [#2872](https://github.com/poanetwork/blockscout/pull/2872), [#2886](https://github.com/poanetwork/blockscout/pull/2886), [#2925](https://github.com/poanetwork/blockscout/pull/2925) - Add "block_hash" to logs, token_transfers and internal transactions and "pending blocks operations" approach
+- [#2926](https://github.com/poanetwork/blockscout/pull/2926) - API endpoint: sum balances except burnt address
+- [#2918](https://github.com/poanetwork/blockscout/pull/2918) - Add tokenID for tokentx API action explicitly
 
 ### Fixes
+- [#2933](https://github.com/poanetwork/blockscout/pull/2933) - Get rid of deadlock in the query to address_current_token_balance table
+- [#2932](https://github.com/poanetwork/blockscout/pull/2932) - fix duplicate websocket connection
+- [#2928](https://github.com/poanetwork/blockscout/pull/2928) - Speedup pending block ops int txs to fetch query
+- [#2924](https://github.com/poanetwork/blockscout/pull/2924) - Speedup address to logs query
+- [#2915](https://github.com/poanetwork/blockscout/pull/2915) - Speedup of blocks_without_reward_query
+- [#2914](https://github.com/poanetwork/blockscout/pull/2914) - Reduce execution time of stream_unfetched_token_instances query
 - [#2908](https://github.com/poanetwork/blockscout/pull/2908) - Fix performance of address page
+- [#2906](https://github.com/poanetwork/blockscout/pull/2906) - fix address sum cache
+- [#2902](https://github.com/poanetwork/blockscout/pull/2902) - Offset in blocks retrieval for average block time
+- [#2900](https://github.com/poanetwork/blockscout/pull/2900) - check fetched instance metadata in multiple places
+- [#2899](https://github.com/poanetwork/blockscout/pull/2899) - fix empty buffered task
+- [#2887](https://github.com/poanetwork/blockscout/pull/2887) - increase chart loading speed
 
 ### Chore
-
 - [#2896](https://github.com/poanetwork/blockscout/pull/2896) - Disable Parity websockets tests
 
 
@@ -14,6 +27,7 @@
 
 ### Features
 - [#2862](https://github.com/poanetwork/blockscout/pull/2862) - Coin total supply from DB API endpoint
+- [#2857](https://github.com/poanetwork/blockscout/pull/2857) - Extend getsourcecode API view with new output fields
 - [#2822](https://github.com/poanetwork/blockscout/pull/2822) - Estimated address count on the main page, if cache is empty
 - [#2821](https://github.com/poanetwork/blockscout/pull/2821) - add autodetection of constructor arguments
 - [#2825](https://github.com/poanetwork/blockscout/pull/2825) - separate token transfers and transactions

apps/block_scout_web/assets/css/components/_dashboard-banner.scss

@@ -73,19 +73,6 @@ $dashboard-banner-chart-axis-font-color: $dashboard-stats-item-value-color !defa
   }
 }
 
-.dashboard-banner-chart-loader {
-  opacity: 0.5;
-  position: absolute;
-  width: calc(100% - 100px);
-  top: 50%;
-  transform: translateY(-75%);
-  left: 50px;
-  right: 50px;
-  transition: opacity .25s ease-in-out;
-  -moz-transition: opacity .25s ease-in-out;
-  -webkit-transition: opacity .25s ease-in-out;
-}
-
 .dashboard-banner-chart-legend {
   display: grid;
   grid-template-columns: 1fr 1fr;

apps/block_scout_web/assets/js/app.js

@@ -43,7 +43,6 @@ import './lib/currency'
 import './lib/from_now'
 import './lib/indexing'
 import './lib/loading_element'
-import './lib/market_history_chart'
 import './lib/pending_transactions_toggle'
 import './lib/pretty_json'
 import './lib/reload_button'

apps/block_scout_web/assets/js/chart-loader.js

@@ -0,0 +1,18 @@
+import $ from 'jquery'
+
+import { formatAllUsdValues, updateAllCalculatedUsdValues } from './lib/currency'
+import { createMarketHistoryChart } from './lib/market_history_chart'
+import { createCoinBalanceHistoryChart } from './lib/coin_balance_history_chart'
+
+(function () {
+  const dashboardChartElement = $('[data-chart="marketHistoryChart"]')[0]
+  const coinBalanceHistoryChartElement = $('[data-chart="coinBalanceHistoryChart"]')[0]
+  if (dashboardChartElement) {
+    window.dashboardChart = createMarketHistoryChart(dashboardChartElement)
+  }
+  if (coinBalanceHistoryChartElement) {
+    window.coinBalanceHistoryChart = createCoinBalanceHistoryChart(coinBalanceHistoryChartElement)
+  }
+  formatAllUsdValues()
+  updateAllCalculatedUsdValues()
+})()

apps/block_scout_web/assets/js/lib/currency.js

@@ -1,8 +1,6 @@
 import $ from 'jquery'
-import humps from 'humps'
 import numeral from 'numeral'
 import { BigNumber } from 'bignumber.js'
-import socket from '../socket'
 
 export function formatUsdValue (value) {
   return `${formatCurrencyValue(value)} USD`
@@ -63,7 +61,3 @@ export function updateAllCalculatedUsdValues (usdExchangeRate) {
   $('[data-usd-unit-price]').each((i, el) => tryUpdateUnitPriceValues(el, usdExchangeRate))
 }
 updateAllCalculatedUsdValues()
-
-export const exchangeRateChannel = socket.channel('exchange_rate:new_rate')
-exchangeRateChannel.join()
-exchangeRateChannel.on('new_rate', (msg) => updateAllCalculatedUsdValues(humps.camelizeKeys(msg).exchangeRate.usdValue))

apps/block_scout_web/assets/js/lib/market_history_chart.js

@@ -80,12 +80,16 @@ function getDataFromLocalStorage (key) {
   return data ? JSON.parse(data) : []
 }
 
+function setDataToLocalStorage (key, data) {
+  window.localStorage.setItem(key, JSON.stringify(data))
+}
+
 function getPriceData (marketHistoryData) {
   if (marketHistoryData.length === 0) {
     return getDataFromLocalStorage('priceData')
   }
   const data = marketHistoryData.map(({ date, closingPrice }) => ({ x: date, y: closingPrice }))
-  window.localStorage.setItem('priceData', JSON.stringify(data))
+  setDataToLocalStorage('priceData', data)
   return data
 }
 
@@ -99,7 +103,7 @@ function getMarketCapData (marketHistoryData, availableSupply) {
       : availableSupply
     return { x: date, y: closingPrice * supply }
   })
-  window.localStorage.setItem('marketCapData', JSON.stringify(data))
+  setDataToLocalStorage('marketCapData', data)
   return data
 }
 
@@ -138,6 +142,15 @@ class MarketHistoryChart {
     }
     this.availableSupply = availableSupply
     config.data.datasets = [this.price, this.marketCap]
+
+    const isChartLoadedKey = 'isChartLoaded'
+    const isChartLoaded = window.sessionStorage.getItem(isChartLoadedKey) === 'true'
+    if (isChartLoaded) {
+      config.options.animation = false
+    } else {
+      window.sessionStorage.setItem(isChartLoadedKey, true)
+    }
+
     this.chart = new Chart(el, config)
   }
 
@@ -156,7 +169,6 @@ class MarketHistoryChart {
 
 export function createMarketHistoryChart (el) {
   const dataPath = el.dataset.market_history_chart_path
-  const $chartLoading = $('[data-chart-loading-message]')
   const $chartError = $('[data-chart-error-message]')
   const chart = new MarketHistoryChart(el, 0, [])
   $(el).show()
@@ -171,15 +183,10 @@ export function createMarketHistoryChart (el) {
       $(el).hide()
       $chartError.show()
     })
-    .always(() => {
-      $chartLoading.css({ opacity: 0 })
-      setTimeout($chartLoading.hide, 1000)
-    })
   return chart
 }
 
 $('[data-chart-error-message]').on('click', _event => {
-  $('[data-chart-loading-message]').show()
   $('[data-chart-error-message]').hide()
   createMarketHistoryChart($('[data-chart="marketHistoryChart"]')[0])
 })

apps/block_scout_web/assets/js/lib/modals.js

@@ -1,5 +1,4 @@
 import $ from 'jquery'
-import Chart from 'chart.js'
 
 $(function () {
   $('.js-become-candidate').on('click', function () {
@@ -43,34 +42,34 @@ $(function () {
   })
 
   function setupStakesProgress (progress, total, modal) {
-    const stakeProgress = $(`${modal} .js-stakes-progress`)
+    // const stakeProgress = $(`${modal} .js-stakes-progress`)
-    const primaryColor = $('.btn-full-primary').css('background-color')
+    // const primaryColor = $('.btn-full-primary').css('background-color')
-    const backgroundColors = [
+    // const backgroundColors = [
-      primaryColor,
+    //   primaryColor,
-      'rgba(202, 199, 226, 0.5)'
+    //   'rgba(202, 199, 226, 0.5)'
-    ]
+    // ]
-    const progressBackground = total - progress
+    // const progressBackground = total - progress
 
-    // eslint-disable-next-line no-unused-vars
+    // // eslint-disable-next-line no-unused-vars
-    const myChart = new Chart(stakeProgress, {
+    // const myChart = new window.Chart(stakeProgress, {
-      type: 'doughnut',
+    //   type: 'doughnut',
-      data: {
+    //   data: {
-        datasets: [{
+    //     datasets: [{
-          data: [progress, progressBackground],
+    //       data: [progress, progressBackground],
-          backgroundColor: backgroundColors,
+    //       backgroundColor: backgroundColors,
-          hoverBackgroundColor: backgroundColors,
+    //       hoverBackgroundColor: backgroundColors,
-          borderWidth: 0
+    //       borderWidth: 0
-        }]
+    //     }]
-      },
+    //   },
-      options: {
+    //   options: {
-        cutoutPercentage: 80,
+    //     cutoutPercentage: 80,
-        legend: {
+    //     legend: {
-          display: false
+    //       display: false
-        },
+    //     },
-        tooltips: {
+    //     tooltips: {
-          enabled: false
+    //       enabled: false
-        }
+    //     }
-      }
+    //   }
-    })
+    // })
   }
 })

apps/block_scout_web/assets/js/pages/address/coin_balances.js

@@ -4,7 +4,6 @@ import humps from 'humps'
 import socket from '../../socket'
 import { connectElements } from '../../lib/redux_helpers.js'
 import { createAsyncLoadStore } from '../../lib/async_listing_load'
-import { createCoinBalanceHistoryChart } from '../../lib/coin_balance_history_chart'
 
 export const initialState = {
   channelDisconnected: false
@@ -61,9 +60,4 @@ if ($('[data-page="coin-balance-history"]').length) {
       msg: humps.camelizeKeys(msg)
     })
   })
-
-  const chartContainer = $('[data-chart="coinBalanceHistoryChart"]')[0]
-  if (chartContainer) {
-    createCoinBalanceHistoryChart(chartContainer)
-  }
 }

apps/block_scout_web/assets/js/pages/chain.js

@@ -7,11 +7,10 @@ import map from 'lodash/map'
 import humps from 'humps'
 import numeral from 'numeral'
 import socket from '../socket'
-import { exchangeRateChannel, formatUsdValue } from '../lib/currency'
+import { updateAllCalculatedUsdValues, formatUsdValue } from '../lib/currency'
 import { createStore, connectElements } from '../lib/redux_helpers.js'
 import { batchChannel, showLoader } from '../lib/utils'
 import listMorph from '../lib/list_morph'
-import { createMarketHistoryChart } from '../lib/market_history_chart'
 
 const BATCH_THRESHOLD = 6
 
@@ -142,8 +141,8 @@ function withMissingBlocks (reducer) {
 let chart
 const elements = {
   '[data-chart="marketHistoryChart"]': {
-    load ($el) {
+    load () {
-      chart = createMarketHistoryChart($el[0])
+      chart = window.dashboardChart
     },
     render ($el, state, oldState) {
       if (!chart || (oldState.availableSupply === state.availableSupply && oldState.marketHistoryData === state.marketHistoryData) || !state.availableSupply) return
@@ -259,10 +258,15 @@ if ($chainDetailsPage.length) {
   loadBlocks(store)
   bindBlockErrorMessage(store)
 
-  exchangeRateChannel.on('new_rate', (msg) => store.dispatch({
+  const exchangeRateChannel = socket.channel('exchange_rate:new_rate')
-    type: 'RECEIVED_NEW_EXCHANGE_RATE',
+  exchangeRateChannel.join()
-    msg: humps.camelizeKeys(msg)
+  exchangeRateChannel.on('new_rate', (msg) => {
-  }))
+    updateAllCalculatedUsdValues(humps.camelizeKeys(msg).exchangeRate.usdValue)
+    store.dispatch({
+      type: 'RECEIVED_NEW_EXCHANGE_RATE',
+      msg: humps.camelizeKeys(msg)
+    })
+  })
 
   const addressesChannel = socket.channel('addresses:new_address')
   addressesChannel.join()

apps/block_scout_web/assets/webpack.config.js

@@ -75,6 +75,7 @@ const appJs =
     entry: {
       app: './js/app.js',
       stakes: './js/pages/stakes.js',
+      'chart-loader': './js/chart-loader.js',
       'non-critical': './css/non-critical.scss'
     },
     output: {

apps/block_scout_web/lib/block_scout_web/controllers/api/rpc/stats_controller.ex

@@ -4,7 +4,7 @@ defmodule BlockScoutWeb.API.RPC.StatsController do
   use Explorer.Schema
 
   alias Explorer.{Chain, ExchangeRates}
-  alias Explorer.Chain.Cache.AddressSum
+  alias Explorer.Chain.Cache.{AddressSum, AddressSumMinusBurnt}
   alias Explorer.Chain.Wei
 
   def tokensupply(conn, params) do
@@ -41,6 +41,16 @@ defmodule BlockScoutWeb.API.RPC.StatsController do
     render(conn, "ethsupply.json", total_supply: cached_wei_total_supply)
   end
 
+  def coinsupply(conn, _params) do
+    cached_coin_total_supply_wei = AddressSumMinusBurnt.get_sum_minus_burnt()
+
+    cached_coin_total_supply =
+      %Wei{value: Decimal.new(cached_coin_total_supply_wei)}
+      |> Wei.to(:ether)
+
+    render(conn, "coinsupply.json", cached_coin_total_supply)
+  end
+
   def ethprice(conn, _params) do
     symbol = Application.get_env(:explorer, :coin)
     rates = ExchangeRates.lookup(symbol)

apps/block_scout_web/lib/block_scout_web/etherscan.ex

@@ -273,6 +273,8 @@ defmodule BlockScoutWeb.Etherscan do
     "result" => "101959776311500000000000000"
   }
 
+  @stats_coinsupply_example_value 101_959_776.3115
+
   @stats_ethprice_example_value %{
     "status" => "1",
     "message" => "OK",
@@ -589,6 +591,12 @@ defmodule BlockScoutWeb.Etherscan do
     example: ~s("Some Token Name")
   }
 
+  @token_id_type %{
+    type: "integer",
+    definition: "id of token",
+    example: ~s("0")
+  }
+
   @token_symbol_type %{
     type: "string",
     definition: "Trading symbol of the token.",
@@ -752,6 +760,7 @@ defmodule BlockScoutWeb.Etherscan do
         example: ~s("663046792267785498951364")
       },
       tokenName: @token_name_type,
+      tokenID: @token_id_type,
       tokenSymbol: @token_symbol_type,
       tokenDecimal: @token_decimal_type,
       transactionIndex: @transaction_index_type,
@@ -1821,7 +1830,7 @@ defmodule BlockScoutWeb.Etherscan do
             message: @message_type,
             result: %{
               type: "integer",
-              description: "The total supply.",
+              description: "The total supply in Wei from DB.",
               example: ~s("101959776311500000000000000")
             }
           }
@@ -1830,6 +1839,30 @@ defmodule BlockScoutWeb.Etherscan do
     ]
   }
 
+  @stats_coinsupply_action %{
+    name: "coinsupply",
+    description: "Get total coin supply from DB minus burnt number.",
+    required_params: [],
+    optional_params: [],
+    responses: [
+      %{
+        code: "200",
+        description: "successful operation",
+        example_value: Jason.encode!(@stats_coinsupply_example_value),
+        model: %{
+          name: "Result",
+          fields: %{
+            result: %{
+              type: "integer",
+              description: "The total supply from DB minus burnt number in coin dimension.",
+              example: 101_959_776.3115
+            }
+          }
+        }
+      }
+    ]
+  }
+
   @stats_ethprice_action %{
     name: "ethprice",
     description: "Get latest price in USD and BTC.",
@@ -2336,6 +2369,7 @@ defmodule BlockScoutWeb.Etherscan do
       @stats_tokensupply_action,
       @stats_ethsupplyexchange_action,
       @stats_ethsupply_action,
+      @stats_coinsupply_action,
       @stats_ethprice_action
     ]
   }

apps/block_scout_web/lib/block_scout_web/notifier.ex

@@ -95,7 +95,7 @@ defmodule BlockScoutWeb.Notifier do
   def handle_event({:chain_event, :internal_transactions, :realtime, internal_transactions}) do
     internal_transactions
     |> Stream.map(
-      &(InternalTransaction
+      &(InternalTransaction.where_nonpending_block()
         |> Repo.get_by(transaction_hash: &1.transaction_hash, index: &1.index)
         |> Repo.preload([:from_address, :to_address, transaction: :block]))
     )

apps/block_scout_web/lib/block_scout_web/templates/address_coin_balance/index.html.eex

@@ -1,4 +1,6 @@
 <section class="container" data-page="coin-balance-history">
+  <script defer src="<%= static_path(@conn, "/js/chart-loader.js") %>"></script>
+
   <%= render BlockScoutWeb.AddressView, "overview.html", assigns %>
 
   <section>

apps/block_scout_web/lib/block_scout_web/templates/chain/show.html.eex

@@ -5,13 +5,6 @@
       <div class="dashboard-banner-network-graph">
         <!-- Graph -->
         <div class="dashboard-banner-chart">
-          <div data-chart-loading-message class="tile tile-muted text-center dashboard-banner-chart-loader">
-            <span class="loading-spinner-small mr-2">
-              <span class="loading-spinner-block-1"></span>
-              <span class="loading-spinner-block-2"></span>
-            </span>
-            <%= gettext("Loading chart") %>...
-          </div>
           <button data-chart-error-message class="alert alert-danger col-12 text-left mt-5" style="display: none;">
             <span><%= gettext("There was a problem loading the chart.") %></span>
           </button>
@@ -79,6 +72,7 @@
       </div>
     </div>
   </div>
+  <script defer data-cfasync="false" src="<%= static_path(@conn, "/js/chart-loader.js") %>"></script>
 </div>
 <section class="container">
   <div class="card card-chain-blocks">

apps/block_scout_web/lib/block_scout_web/templates/layout/app.html.eex

@@ -4,6 +4,7 @@
     <meta charset="utf-8">
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <meta name="viewport" content="width=device-width, initial-scale=1">
+
     <link rel="stylesheet" href="<%= static_path(@conn, "/css/app.css") %>">
     <link rel="preload" href="<%= static_path(@conn, "/css/non-critical.css") %>" as="style" onload="this.onload=null;this.rel='stylesheet'">
     <noscript><link rel="stylesheet" href="<%= static_path(@conn, "/css/non-critical.css") %>"></noscript>
@@ -19,6 +20,18 @@
     <meta name="theme-color" content="#ffffff">
 
     <%= render_existing(@view_module, "_metatags.html", assigns) || render("_default_title.html") %>
+
+    <script defer data-cfasync="false">
+      window.localized = {
+        'Blocks Indexed': '<%= gettext("Blocks Indexed") %>',
+        'Block Processing': '<%= gettext("Block Mined, awaiting import...") %>',
+        'Indexing Tokens': '<%= gettext("Indexing Tokens") %>',
+        'Less than': '<%= gettext("Less than") %>',
+        'Market Cap': '<%= gettext("Market Cap") %>',
+        'Price': '<%= gettext("Price") %>',
+        'Ether': '<%= gettext("Ether") %>'
+      }
+    </script>
   </head>
 
   <body>
@@ -51,18 +64,7 @@
       </main>
       <%= render BlockScoutWeb.LayoutView, "_footer.html", assigns %>
     </div>
-    <script>
+    <script defer data-cfasync="false" src="<%= static_path(@conn, "/js/app.js") %>"></script>
-      window.localized = {
-        'Blocks Indexed': '<%= gettext("Blocks Indexed") %>',
-        'Block Processing': '<%= gettext("Block Mined, awaiting import...") %>',
-        'Indexing Tokens': '<%= gettext("Indexing Tokens") %>',
-        'Less than': '<%= gettext("Less than") %>',
-        'Market Cap': '<%= gettext("Market Cap") %>',
-        'Price': '<%= gettext("Price") %>',
-        'Ether': '<%= gettext("Ether") %>'
-      }
-    </script>
-    <script src="<%= static_path(@conn, "/js/app.js") %>"></script>
     <%= render_existing(@view_module, "scripts.html", assigns) %>
   </body>
 </html>

apps/block_scout_web/lib/block_scout_web/views/api/rpc/address_view.ex

@@ -121,7 +121,7 @@ defmodule BlockScoutWeb.API.RPC.AddressView do
     }
   end
 
-  defp prepare_token_transfer(token_transfer) do
+  defp prepare_common_token_transfer(token_transfer) do
     %{
       "blockNumber" => to_string(token_transfer.block_number),
       "timeStamp" => to_string(DateTime.to_unix(token_transfer.block_timestamp)),
@@ -132,7 +132,6 @@ defmodule BlockScoutWeb.API.RPC.AddressView do
       "contractAddress" => to_string(token_transfer.token_contract_address_hash),
       "to" => to_string(token_transfer.to_address_hash),
       "logIndex" => to_string(token_transfer.token_log_index),
-      "value" => get_token_value(token_transfer),
       "tokenName" => token_transfer.token_name,
       "tokenSymbol" => token_transfer.token_symbol,
       "tokenDecimal" => to_string(token_transfer.token_decimals),
@@ -146,12 +145,20 @@ defmodule BlockScoutWeb.API.RPC.AddressView do
     }
   end
 
-  defp get_token_value(%{token_type: "ERC-721"} = token_transfer) do
+  defp prepare_token_transfer(%{token_type: "ERC-721"} = token_transfer) do
-    to_string(token_transfer.token_id)
+    token_transfer
+    |> prepare_common_token_transfer()
+    |> Map.put_new(:tokenID, token_transfer.token_id)
+  end
+
+  defp prepare_token_transfer(%{token_type: "ERC-20"} = token_transfer) do
+    token_transfer
+    |> prepare_common_token_transfer()
+    |> Map.put_new(:value, to_string(token_transfer.amount))
   end
 
-  defp get_token_value(token_transfer) do
+  defp prepare_token_transfer(token_transfer) do
-    to_string(token_transfer.amount)
+    prepare_common_token_transfer(token_transfer)
   end
 
   defp prepare_block(block) do

apps/block_scout_web/lib/block_scout_web/views/api/rpc/contract_view.ex

@@ -4,6 +4,8 @@ defmodule BlockScoutWeb.API.RPC.ContractView do
   alias BlockScoutWeb.API.RPC.RPCView
   alias Explorer.Chain.{Address, DecompiledSmartContract, SmartContract}
 
+  defguardp is_empty_string(input) when input == "" or input == nil
+
   def render("listcontracts.json", %{contracts: contracts}) do
     contracts = Enum.map(contracts, &prepare_contract/1)
 
@@ -35,7 +37,11 @@ defmodule BlockScoutWeb.API.RPC.ContractView do
       "CompilerVersion" => "",
       "DecompiledSourceCode" => "",
       "DecompilerVersion" => decompiler_version(nil),
-      "OptimizationUsed" => ""
+      "OptimizationUsed" => "",
+      "OptimizationRuns" => "",
+      "EVMVersion" => "",
+      "ConstructorArguments" => "",
+      "ExternalLibraries" => ""
     }
   end
 
@@ -43,6 +49,68 @@ defmodule BlockScoutWeb.API.RPC.ContractView do
     decompiled_smart_contract = latest_decompiled_smart_contract(address.decompiled_smart_contracts)
     contract = address.smart_contract || %{}
 
+    optimization = Map.get(contract, :optimization, "")
+
+    contract_output = %{
+      "Address" => to_string(address.hash)
+    }
+
+    contract_output
+    |> set_decompiled_contract_data(decompiled_smart_contract)
+    |> set_optimization_runs(contract, optimization)
+    |> set_constructor_arguments(contract)
+    |> set_external_libraries(contract)
+    |> set_verified_contract_data(contract, address, optimization)
+  end
+
+  defp set_decompiled_contract_data(contract_output, decompiled_smart_contract) do
+    if decompiled_smart_contract do
+      contract_output
+      |> Map.put_new(:DecompiledSourceCode, decompiled_source_code(decompiled_smart_contract))
+      |> Map.put_new(:DecompilerVersion, decompiler_version(decompiled_smart_contract))
+    else
+      contract_output
+    end
+  end
+
+  defp set_optimization_runs(contract_output, contract, optimization) do
+    optimization_runs = Map.get(contract, :optimization_runs, "")
+
+    if optimization && optimization != "" do
+      contract_output
+      |> Map.put_new(:OptimizationRuns, optimization_runs)
+    else
+      contract_output
+    end
+  end
+
+  defp set_constructor_arguments(contract_output, %{constructor_arguments: arguments}) when is_empty_string(arguments),
+    do: contract_output
+
+  defp set_constructor_arguments(contract_output, %{constructor_arguments: arguments}) do
+    contract_output
+    |> Map.put_new(:ConstructorArguments, arguments)
+  end
+
+  defp set_constructor_arguments(contract_output, _), do: contract_output
+
+  defp set_external_libraries(contract_output, contract) do
+    external_libraries = Map.get(contract, :external_libraries, [])
+
+    if Enum.count(external_libraries) > 0 do
+      external_libraries_without_id =
+        Enum.map(external_libraries, fn %{name: name, address_hash: address_hash} ->
+          %{"name" => name, "address_hash" => address_hash}
+        end)
+
+      contract_output
+      |> Map.put_new(:ExternalLibraries, external_libraries_without_id)
+    else
+      contract_output
+    end
+  end
+
+  defp set_verified_contract_data(contract_output, contract, address, optimization) do
     contract_abi =
       if is_nil(address.smart_contract) do
         "Contract source code not verified"
@@ -51,27 +119,28 @@ defmodule BlockScoutWeb.API.RPC.ContractView do
       end
 
     contract_optimization =
-      case Map.get(contract, :optimization, "") do
+      case optimization do
         true ->
-          "1"
+          "true"
 
         false ->
-          "0"
+          "false"
 
         "" ->
           ""
       end
 
-    %{
+    if Map.equal?(contract, %{}) do
-      "Address" => to_string(address.hash),
+      contract_output
-      "SourceCode" => Map.get(contract, :contract_source_code, ""),
+    else
-      "ABI" => contract_abi,
+      contract_output
-      "ContractName" => Map.get(contract, :name, ""),
+      |> Map.put_new(:SourceCode, Map.get(contract, :contract_source_code, ""))
-      "DecompiledSourceCode" => decompiled_source_code(decompiled_smart_contract),
+      |> Map.put_new(:ABI, contract_abi)
-      "DecompilerVersion" => decompiler_version(decompiled_smart_contract),
+      |> Map.put_new(:ContractName, Map.get(contract, :name, ""))
-      "CompilerVersion" => Map.get(contract, :compiler_version, ""),
+      |> Map.put_new(:CompilerVersion, Map.get(contract, :compiler_version, ""))
-      "OptimizationUsed" => contract_optimization
+      |> Map.put_new(:OptimizationUsed, contract_optimization)
-    }
+      |> Map.put_new(:EVMVersion, Map.get(contract, :evm_version, ""))
+    end
   end
 
   defp prepare_contract(%Address{
@@ -80,10 +149,7 @@ defmodule BlockScoutWeb.API.RPC.ContractView do
        }) do
     %{
       "Address" => to_string(hash),
-      "ABI" => "Contract source code not verified",
+      "ABI" => "Contract source code not verified"
-      "ContractName" => "",
-      "CompilerVersion" => "",
-      "OptimizationUsed" => ""
     }
   end
 

apps/block_scout_web/lib/block_scout_web/views/api/rpc/rpc_view.ex

@@ -9,6 +9,15 @@ defmodule BlockScoutWeb.API.RPC.RPCView do
     }
   end
 
+  def render("show_value.json", data) do
+    {value, _} =
+      data
+      |> Decimal.to_string(:normal)
+      |> Float.parse()
+
+    value
+  end
+
   def render("error.json", %{error: message} = assigns) do
     %{
       "status" => "0",

apps/block_scout_web/lib/block_scout_web/views/api/rpc/stats_view.ex

@@ -15,6 +15,10 @@ defmodule BlockScoutWeb.API.RPC.StatsView do
     RPCView.render("show.json", data: total_supply)
   end
 
+  def render("coinsupply.json", total_supply) do
+    RPCView.render("show_value.json", total_supply)
+  end
+
   def render("ethprice.json", %{rates: rates}) do
     RPCView.render("show.json", data: prepare_rates(rates))
   end

apps/block_scout_web/priv/gettext/default.pot

@@ -72,7 +72,7 @@ msgid "(query)"
 msgstr ""
 
 #, elixir-format
-#: lib/block_scout_web/templates/layout/app.html.eex:42
+#: lib/block_scout_web/templates/layout/app.html.eex:55
 msgid "- We're indexing this chain right now. Some of the counts may be inaccurate."
 msgstr ""
 
@@ -150,7 +150,7 @@ msgid "Anything not in this list is not supported. Click on the method to be tak
 msgstr ""
 
 #, elixir-format
-#: lib/block_scout_web/templates/chain/show.html.eex:47
+#: lib/block_scout_web/templates/chain/show.html.eex:40
 msgid "Average block time"
 msgstr ""
 
@@ -165,7 +165,7 @@ msgid "Balance"
 msgstr ""
 
 #, elixir-format
-#: lib/block_scout_web/templates/address_coin_balance/index.html.eex:14
+#: lib/block_scout_web/templates/address_coin_balance/index.html.eex:16
 msgid "Balances"
 msgstr ""
 
@@ -208,7 +208,7 @@ msgid "Block Height: %{height}"
 msgstr ""
 
 #, elixir-format
-#: lib/block_scout_web/templates/layout/app.html.eex:57
+#: lib/block_scout_web/templates/layout/app.html.eex:27
 msgid "Block Mined, awaiting import..."
 msgstr ""
 
@@ -233,14 +233,14 @@ msgid "BlockScout provides analytics data, API, and Smart Contract tools for the
 msgstr ""
 
 #, elixir-format
-#: lib/block_scout_web/templates/chain/show.html.eex:87
+#: lib/block_scout_web/templates/chain/show.html.eex:81
 #: lib/block_scout_web/templates/layout/_topnav.html.eex:31
 #: lib/block_scout_web/templates/layout/_topnav.html.eex:35
 msgid "Blocks"
 msgstr ""
 
 #, elixir-format
-#: lib/block_scout_web/templates/layout/app.html.eex:56
+#: lib/block_scout_web/templates/layout/app.html.eex:26
 msgid "Blocks Indexed"
 msgstr ""
 
@@ -304,7 +304,7 @@ msgid "Connection Lost"
 msgstr ""
 
 #, elixir-format
-#: lib/block_scout_web/templates/address_coin_balance/index.html.eex:10
+#: lib/block_scout_web/templates/address_coin_balance/index.html.eex:12
 #: lib/block_scout_web/templates/block/index.html.eex:6
 msgid "Connection Lost, click to load newer blocks"
 msgstr ""
@@ -643,7 +643,7 @@ msgstr ""
 #, elixir-format
 #: lib/block_scout_web/templates/address/_balance_card.html.eex:15
 #: lib/block_scout_web/templates/internal_transaction/_tile.html.eex:20
-#: lib/block_scout_web/templates/layout/app.html.eex:62
+#: lib/block_scout_web/templates/layout/app.html.eex:32
 #: lib/block_scout_web/templates/transaction/_pending_tile.html.eex:20
 #: lib/block_scout_web/templates/transaction/_tile.html.eex:29
 #: lib/block_scout_web/templates/transaction/overview.html.eex:180
@@ -966,7 +966,7 @@ msgid "Last Balance Update: Block #"
 msgstr ""
 
 #, elixir-format
-#: lib/block_scout_web/templates/layout/app.html.eex:59
+#: lib/block_scout_web/templates/layout/app.html.eex:29
 msgid "Less than"
 msgstr ""
 
@@ -1033,8 +1033,8 @@ msgid "Mainnet"
 msgstr ""
 
 #, elixir-format
-#: lib/block_scout_web/templates/chain/show.html.eex:31
+#: lib/block_scout_web/templates/chain/show.html.eex:24
-#: lib/block_scout_web/templates/layout/app.html.eex:60
+#: lib/block_scout_web/templates/layout/app.html.eex:30
 #: lib/block_scout_web/views/address_view.ex:126
 #: lib/block_scout_web/views/address_view.ex:126
 msgid "Market Cap"
@@ -1067,7 +1067,7 @@ msgid "More internal transactions have come in"
 msgstr ""
 
 #, elixir-format
-#: lib/block_scout_web/templates/chain/show.html.eex:148
+#: lib/block_scout_web/templates/chain/show.html.eex:142
 #: lib/block_scout_web/templates/pending_transaction/index.html.eex:10
 #: lib/block_scout_web/templates/transaction/index.html.eex:10
 msgid "More transactions have come in"
@@ -1169,8 +1169,8 @@ msgid "Potential matches from our contract method database:"
 msgstr ""
 
 #, elixir-format
-#: lib/block_scout_web/templates/chain/show.html.eex:24
+#: lib/block_scout_web/templates/chain/show.html.eex:17
-#: lib/block_scout_web/templates/layout/app.html.eex:61
+#: lib/block_scout_web/templates/layout/app.html.eex:31
 msgid "Price"
 msgstr ""
 
@@ -1305,7 +1305,7 @@ msgid "Block Rewards"
 msgstr ""
 
 #, elixir-format
-#: lib/block_scout_web/templates/address_coin_balance/index.html.eex:34
+#: lib/block_scout_web/templates/address_coin_balance/index.html.eex:36
 #: lib/block_scout_web/templates/address_internal_transaction/index.html.eex:61
 #: lib/block_scout_web/templates/address_logs/index.html.eex:21
 #: lib/block_scout_web/templates/address_token/index.html.eex:13
@@ -1313,7 +1313,7 @@ msgstr ""
 #: lib/block_scout_web/templates/address_transaction/index.html.eex:59
 #: lib/block_scout_web/templates/address_validation/index.html.eex:22
 #: lib/block_scout_web/templates/block_transaction/index.html.eex:23
-#: lib/block_scout_web/templates/chain/show.html.eex:91
+#: lib/block_scout_web/templates/chain/show.html.eex:85
 #: lib/block_scout_web/templates/pending_transaction/index.html.eex:19
 #: lib/block_scout_web/templates/tokens/holder/index.html.eex:20
 #: lib/block_scout_web/templates/tokens/instance/transfer/index.html.eex:21
@@ -1327,7 +1327,7 @@ msgid "Something went wrong, click to reload."
 msgstr ""
 
 #, elixir-format
-#: lib/block_scout_web/templates/chain/show.html.eex:154
+#: lib/block_scout_web/templates/chain/show.html.eex:148
 msgid "Something went wrong, click to retry."
 msgstr ""
 
@@ -1392,7 +1392,7 @@ msgid "There are no transactions."
 msgstr ""
 
 #, elixir-format
-#: lib/block_scout_web/templates/address_coin_balance/index.html.eex:39
+#: lib/block_scout_web/templates/address_coin_balance/index.html.eex:41
 msgid "There is no coin history for this address."
 msgstr ""
 
@@ -1402,8 +1402,8 @@ msgid "There is no decompilded contracts for this address."
 msgstr ""
 
 #, elixir-format
-#: lib/block_scout_web/templates/address_coin_balance/index.html.eex:25
+#: lib/block_scout_web/templates/address_coin_balance/index.html.eex:27
-#: lib/block_scout_web/templates/chain/show.html.eex:16
+#: lib/block_scout_web/templates/chain/show.html.eex:9
 msgid "There was a problem loading the chart."
 msgstr ""
 
@@ -1456,7 +1456,7 @@ msgid "Total Supply"
 msgstr ""
 
 #, elixir-format
-#: lib/block_scout_web/templates/chain/show.html.eex:64
+#: lib/block_scout_web/templates/chain/show.html.eex:57
 msgid "Total blocks"
 msgstr ""
 
@@ -1597,12 +1597,12 @@ msgid "Version"
 msgstr ""
 
 #, elixir-format
-#: lib/block_scout_web/templates/chain/show.html.eex:86
+#: lib/block_scout_web/templates/chain/show.html.eex:80
 msgid "View All Blocks"
 msgstr ""
 
 #, elixir-format
-#: lib/block_scout_web/templates/chain/show.html.eex:144
+#: lib/block_scout_web/templates/chain/show.html.eex:138
 msgid "View All Transactions"
 msgstr ""
 
@@ -1644,7 +1644,7 @@ msgid "WEI"
 msgstr ""
 
 #, elixir-format
-#: lib/block_scout_web/templates/chain/show.html.eex:72
+#: lib/block_scout_web/templates/chain/show.html.eex:65
 msgid "Wallet addresses"
 msgstr ""
 
@@ -1755,13 +1755,12 @@ msgid "If it still does not show up after 1 hour, please check with your sender/
 msgstr ""
 
 #, elixir-format
-#: lib/block_scout_web/templates/layout/app.html.eex:58
+#: lib/block_scout_web/templates/layout/app.html.eex:28
 msgid "Indexing Tokens"
 msgstr ""
 
 #, elixir-format
-#: lib/block_scout_web/templates/address_coin_balance/index.html.eex:22
+#: lib/block_scout_web/templates/address_coin_balance/index.html.eex:24
-#: lib/block_scout_web/templates/chain/show.html.eex:13
 msgid "Loading chart"
 msgstr ""
 
@@ -1783,7 +1782,7 @@ msgid "Module"
 msgstr ""
 
 #, elixir-format
-#: lib/block_scout_web/templates/chain/show.html.eex:56
+#: lib/block_scout_web/templates/chain/show.html.eex:49
 msgid "Total transactions"
 msgstr ""
 
@@ -1867,7 +1866,7 @@ msgstr ""
 #: lib/block_scout_web/templates/address_transaction/index.html.eex:15
 #: lib/block_scout_web/templates/block_transaction/index.html.eex:10
 #: lib/block_scout_web/templates/block_transaction/index.html.eex:18
-#: lib/block_scout_web/templates/chain/show.html.eex:145
+#: lib/block_scout_web/templates/chain/show.html.eex:139
 #: lib/block_scout_web/templates/layout/_topnav.html.eex:50
 #: lib/block_scout_web/views/address_view.ex:306
 msgid "Transactions"

apps/block_scout_web/priv/gettext/en/LC_MESSAGES/default.po

@@ -72,7 +72,7 @@ msgid "(query)"
 msgstr ""
 
 #, elixir-format
-#: lib/block_scout_web/templates/layout/app.html.eex:42
+#: lib/block_scout_web/templates/layout/app.html.eex:55
 msgid "- We're indexing this chain right now. Some of the counts may be inaccurate."
 msgstr ""
 
@@ -150,7 +150,7 @@ msgid "Anything not in this list is not supported. Click on the method to be tak
 msgstr ""
 
 #, elixir-format
-#: lib/block_scout_web/templates/chain/show.html.eex:47
+#: lib/block_scout_web/templates/chain/show.html.eex:40
 msgid "Average block time"
 msgstr ""
 
@@ -165,7 +165,7 @@ msgid "Balance"
 msgstr ""
 
 #, elixir-format
-#: lib/block_scout_web/templates/address_coin_balance/index.html.eex:14
+#: lib/block_scout_web/templates/address_coin_balance/index.html.eex:16
 msgid "Balances"
 msgstr ""
 
@@ -208,7 +208,7 @@ msgid "Block Height: %{height}"
 msgstr ""
 
 #, elixir-format
-#: lib/block_scout_web/templates/layout/app.html.eex:57
+#: lib/block_scout_web/templates/layout/app.html.eex:27
 msgid "Block Mined, awaiting import..."
 msgstr ""
 
@@ -233,14 +233,14 @@ msgid "BlockScout provides analytics data, API, and Smart Contract tools for the
 msgstr ""
 
 #, elixir-format
-#: lib/block_scout_web/templates/chain/show.html.eex:87
+#: lib/block_scout_web/templates/chain/show.html.eex:81
 #: lib/block_scout_web/templates/layout/_topnav.html.eex:31
 #: lib/block_scout_web/templates/layout/_topnav.html.eex:35
 msgid "Blocks"
 msgstr ""
 
 #, elixir-format
-#: lib/block_scout_web/templates/layout/app.html.eex:56
+#: lib/block_scout_web/templates/layout/app.html.eex:26
 msgid "Blocks Indexed"
 msgstr ""
 
@@ -304,7 +304,7 @@ msgid "Connection Lost"
 msgstr ""
 
 #, elixir-format
-#: lib/block_scout_web/templates/address_coin_balance/index.html.eex:10
+#: lib/block_scout_web/templates/address_coin_balance/index.html.eex:12
 #: lib/block_scout_web/templates/block/index.html.eex:6
 msgid "Connection Lost, click to load newer blocks"
 msgstr ""
@@ -643,7 +643,7 @@ msgstr ""
 #, elixir-format
 #: lib/block_scout_web/templates/address/_balance_card.html.eex:15
 #: lib/block_scout_web/templates/internal_transaction/_tile.html.eex:20
-#: lib/block_scout_web/templates/layout/app.html.eex:62
+#: lib/block_scout_web/templates/layout/app.html.eex:32
 #: lib/block_scout_web/templates/transaction/_pending_tile.html.eex:20
 #: lib/block_scout_web/templates/transaction/_tile.html.eex:29
 #: lib/block_scout_web/templates/transaction/overview.html.eex:180
@@ -966,7 +966,7 @@ msgid "Last Balance Update: Block #"
 msgstr ""
 
 #, elixir-format
-#: lib/block_scout_web/templates/layout/app.html.eex:59
+#: lib/block_scout_web/templates/layout/app.html.eex:29
 msgid "Less than"
 msgstr ""
 
@@ -1033,8 +1033,8 @@ msgid "Mainnet"
 msgstr ""
 
 #, elixir-format
-#: lib/block_scout_web/templates/chain/show.html.eex:31
+#: lib/block_scout_web/templates/chain/show.html.eex:24
-#: lib/block_scout_web/templates/layout/app.html.eex:60
+#: lib/block_scout_web/templates/layout/app.html.eex:30
 #: lib/block_scout_web/views/address_view.ex:126
 #: lib/block_scout_web/views/address_view.ex:126
 msgid "Market Cap"
@@ -1067,7 +1067,7 @@ msgid "More internal transactions have come in"
 msgstr ""
 
 #, elixir-format
-#: lib/block_scout_web/templates/chain/show.html.eex:148
+#: lib/block_scout_web/templates/chain/show.html.eex:142
 #: lib/block_scout_web/templates/pending_transaction/index.html.eex:10
 #: lib/block_scout_web/templates/transaction/index.html.eex:10
 msgid "More transactions have come in"
@@ -1169,8 +1169,8 @@ msgid "Potential matches from our contract method database:"
 msgstr ""
 
 #, elixir-format
-#: lib/block_scout_web/templates/chain/show.html.eex:24
+#: lib/block_scout_web/templates/chain/show.html.eex:17
-#: lib/block_scout_web/templates/layout/app.html.eex:61
+#: lib/block_scout_web/templates/layout/app.html.eex:31
 msgid "Price"
 msgstr ""
 
@@ -1305,7 +1305,7 @@ msgid "Block Rewards"
 msgstr ""
 
 #, elixir-format
-#: lib/block_scout_web/templates/address_coin_balance/index.html.eex:34
+#: lib/block_scout_web/templates/address_coin_balance/index.html.eex:36
 #: lib/block_scout_web/templates/address_internal_transaction/index.html.eex:61
 #: lib/block_scout_web/templates/address_logs/index.html.eex:21
 #: lib/block_scout_web/templates/address_token/index.html.eex:13
@@ -1313,7 +1313,7 @@ msgstr ""
 #: lib/block_scout_web/templates/address_transaction/index.html.eex:59
 #: lib/block_scout_web/templates/address_validation/index.html.eex:22
 #: lib/block_scout_web/templates/block_transaction/index.html.eex:23
-#: lib/block_scout_web/templates/chain/show.html.eex:91
+#: lib/block_scout_web/templates/chain/show.html.eex:85
 #: lib/block_scout_web/templates/pending_transaction/index.html.eex:19
 #: lib/block_scout_web/templates/tokens/holder/index.html.eex:20
 #: lib/block_scout_web/templates/tokens/instance/transfer/index.html.eex:21
@@ -1327,7 +1327,7 @@ msgid "Something went wrong, click to reload."
 msgstr ""
 
 #, elixir-format
-#: lib/block_scout_web/templates/chain/show.html.eex:154
+#: lib/block_scout_web/templates/chain/show.html.eex:148
 msgid "Something went wrong, click to retry."
 msgstr ""
 
@@ -1392,7 +1392,7 @@ msgid "There are no transactions."
 msgstr ""
 
 #, elixir-format
-#: lib/block_scout_web/templates/address_coin_balance/index.html.eex:39
+#: lib/block_scout_web/templates/address_coin_balance/index.html.eex:41
 msgid "There is no coin history for this address."
 msgstr ""
 
@@ -1402,8 +1402,8 @@ msgid "There is no decompilded contracts for this address."
 msgstr ""
 
 #, elixir-format
-#: lib/block_scout_web/templates/address_coin_balance/index.html.eex:25
+#: lib/block_scout_web/templates/address_coin_balance/index.html.eex:27
-#: lib/block_scout_web/templates/chain/show.html.eex:16
+#: lib/block_scout_web/templates/chain/show.html.eex:9
 msgid "There was a problem loading the chart."
 msgstr ""
 
@@ -1456,7 +1456,7 @@ msgid "Total Supply"
 msgstr ""
 
 #, elixir-format
-#: lib/block_scout_web/templates/chain/show.html.eex:64
+#: lib/block_scout_web/templates/chain/show.html.eex:57
 msgid "Total blocks"
 msgstr ""
 
@@ -1597,12 +1597,12 @@ msgid "Version"
 msgstr ""
 
 #, elixir-format
-#: lib/block_scout_web/templates/chain/show.html.eex:86
+#: lib/block_scout_web/templates/chain/show.html.eex:80
 msgid "View All Blocks"
 msgstr ""
 
 #, elixir-format
-#: lib/block_scout_web/templates/chain/show.html.eex:144
+#: lib/block_scout_web/templates/chain/show.html.eex:138
 msgid "View All Transactions"
 msgstr ""
 
@@ -1644,7 +1644,7 @@ msgid "WEI"
 msgstr ""
 
 #, elixir-format
-#: lib/block_scout_web/templates/chain/show.html.eex:72
+#: lib/block_scout_web/templates/chain/show.html.eex:65
 msgid "Wallet addresses"
 msgstr ""
 
@@ -1755,13 +1755,12 @@ msgid "If it still does not show up after 1 hour, please check with your sender/
 msgstr ""
 
 #, elixir-format
-#: lib/block_scout_web/templates/layout/app.html.eex:58
+#: lib/block_scout_web/templates/layout/app.html.eex:28
 msgid "Indexing Tokens"
 msgstr ""
 
 #, elixir-format
-#: lib/block_scout_web/templates/address_coin_balance/index.html.eex:22
+#: lib/block_scout_web/templates/address_coin_balance/index.html.eex:24
-#: lib/block_scout_web/templates/chain/show.html.eex:13
 msgid "Loading chart"
 msgstr ""
 
@@ -1783,7 +1782,7 @@ msgid "Module"
 msgstr ""
 
 #, elixir-format
-#: lib/block_scout_web/templates/chain/show.html.eex:56
+#: lib/block_scout_web/templates/chain/show.html.eex:49
 msgid "Total transactions"
 msgstr ""
 
@@ -1867,7 +1866,7 @@ msgstr ""
 #: lib/block_scout_web/templates/address_transaction/index.html.eex:15
 #: lib/block_scout_web/templates/block_transaction/index.html.eex:10
 #: lib/block_scout_web/templates/block_transaction/index.html.eex:18
-#: lib/block_scout_web/templates/chain/show.html.eex:145
+#: lib/block_scout_web/templates/chain/show.html.eex:139
 #: lib/block_scout_web/templates/layout/_topnav.html.eex:50
 #: lib/block_scout_web/views/address_view.ex:306
 msgid "Transactions"

apps/block_scout_web/test/block_scout_web/channels/address_channel_test.exs

@@ -134,7 +134,15 @@ defmodule BlockScoutWeb.AddressChannelTest do
         |> insert(from_address: address)
         |> with_block()
 
-      internal_transaction = insert(:internal_transaction, transaction: transaction, from_address: address, index: 0)
+      internal_transaction =
+        insert(
+          :internal_transaction,
+          transaction: transaction,
+          from_address: address,
+          index: 0,
+          block_hash: transaction.block_hash,
+          block_index: 0
+        )
 
       Notifier.handle_event({:chain_event, :internal_transactions, :realtime, [internal_transaction]})
 
@@ -158,7 +166,14 @@ defmodule BlockScoutWeb.AddressChannelTest do
         |> insert(to_address: address)
         |> with_block()
 
-      internal_transaction = insert(:internal_transaction, transaction: transaction, to_address: address, index: 0)
+      internal_transaction =
+        insert(:internal_transaction,
+          transaction: transaction,
+          to_address: address,
+          index: 0,
+          block_hash: transaction.block_hash,
+          block_index: 0
+        )
 
       Notifier.handle_event({:chain_event, :internal_transactions, :realtime, [internal_transaction]})
 
@@ -186,7 +201,14 @@ defmodule BlockScoutWeb.AddressChannelTest do
         |> with_block()
 
       internal_transaction =
-        insert(:internal_transaction, transaction: transaction, from_address: address, to_address: address, index: 0)
+        insert(:internal_transaction,
+          transaction: transaction,
+          from_address: address,
+          to_address: address,
+          index: 0,
+          block_hash: transaction.block_hash,
+          block_index: 0
+        )
 
       Notifier.handle_event({:chain_event, :internal_transactions, :realtime, [internal_transaction]})
 

apps/block_scout_web/test/block_scout_web/controllers/address_contract_controller_test.exs

@@ -35,13 +35,15 @@ defmodule BlockScoutWeb.AddressContractControllerTest do
     test "successfully renders the page when the address is a contract", %{conn: conn} do
       address = insert(:address, contract_code: Factory.data("contract_code"), smart_contract: nil)
 
-      transaction = insert(:transaction, from_address: address)
+      transaction = insert(:transaction, from_address: address) |> with_block()
 
       insert(
         :internal_transaction_create,
         index: 0,
         transaction: transaction,
-        created_contract_address: address
+        created_contract_address: address,
+        block_hash: transaction.block_hash,
+        block_index: 0
       )
 
       conn = get(conn, address_contract_path(BlockScoutWeb.Endpoint, :index, address))

apps/block_scout_web/test/block_scout_web/controllers/address_internal_transaction_controller_test.exs

@@ -44,7 +44,9 @@ defmodule BlockScoutWeb.AddressInternalTransactionControllerTest do
           from_address: address,
           index: 1,
           block_number: transaction.block_number,
-          transaction_index: transaction.index
+          transaction_index: transaction.index,
+          block_hash: transaction.block_hash,
+          block_index: 1
         )
 
       to_internal_transaction =
@@ -53,7 +55,9 @@ defmodule BlockScoutWeb.AddressInternalTransactionControllerTest do
           to_address: address,
           index: 2,
           block_number: transaction.block_number,
-          transaction_index: transaction.index
+          transaction_index: transaction.index,
+          block_hash: transaction.block_hash,
+          block_index: 2
         )
 
       path = address_internal_transaction_path(conn, :index, address, %{"type" => "JSON"})
@@ -83,7 +87,9 @@ defmodule BlockScoutWeb.AddressInternalTransactionControllerTest do
           from_address: address,
           index: 1,
           block_number: transaction.block_number,
-          transaction_index: transaction.index
+          transaction_index: transaction.index,
+          block_hash: transaction.block_hash,
+          block_index: 1
         )
 
       to_internal_transaction =
@@ -92,7 +98,9 @@ defmodule BlockScoutWeb.AddressInternalTransactionControllerTest do
           to_address: address,
           index: 2,
           block_number: transaction.block_number,
-          transaction_index: transaction.index
+          transaction_index: transaction.index,
+          block_hash: transaction.block_hash,
+          block_index: 2
         )
 
       path = address_internal_transaction_path(conn, :index, address, %{"filter" => "from", "type" => "JSON"})
@@ -125,7 +133,9 @@ defmodule BlockScoutWeb.AddressInternalTransactionControllerTest do
           from_address: address,
           index: 1,
           block_number: transaction.block_number,
-          transaction_index: transaction.index
+          transaction_index: transaction.index,
+          block_hash: transaction.block_hash,
+          block_index: 1
         )
 
       to_internal_transaction =
@@ -134,7 +144,9 @@ defmodule BlockScoutWeb.AddressInternalTransactionControllerTest do
           to_address: address,
           index: 2,
           block_number: transaction.block_number,
-          transaction_index: transaction.index
+          transaction_index: transaction.index,
+          block_hash: transaction.block_hash,
+          block_index: 2
         )
 
       path = address_internal_transaction_path(conn, :index, address, %{"filter" => "to", "type" => "JSON"})
@@ -167,7 +179,9 @@ defmodule BlockScoutWeb.AddressInternalTransactionControllerTest do
           from_address: address,
           index: 1,
           block_number: transaction.block_number,
-          transaction_index: transaction.index
+          transaction_index: transaction.index,
+          block_hash: transaction.block_hash,
+          block_index: 1
         )
 
       to_internal_transaction =
@@ -177,7 +191,9 @@ defmodule BlockScoutWeb.AddressInternalTransactionControllerTest do
           created_contract_address: address,
           index: 2,
           block_number: transaction.block_number,
-          transaction_index: transaction.index
+          transaction_index: transaction.index,
+          block_hash: transaction.block_hash,
+          block_index: 2
         )
 
       path = address_internal_transaction_path(conn, :index, address, %{"filter" => "to", "type" => "JSON"})
@@ -226,7 +242,9 @@ defmodule BlockScoutWeb.AddressInternalTransactionControllerTest do
             from_address: address,
             index: index,
             block_number: transaction_1.block_number,
-            transaction_index: transaction_1.index
+            transaction_index: transaction_1.index,
+            block_hash: a_block.hash,
+            block_index: index
           )
         end)
 
@@ -239,7 +257,9 @@ defmodule BlockScoutWeb.AddressInternalTransactionControllerTest do
             from_address: address,
             index: index,
             block_number: transaction_2.block_number,
-            transaction_index: transaction_2.index
+            transaction_index: transaction_2.index,
+            block_hash: a_block.hash,
+            block_index: 20 + index
           )
         end)
 
@@ -252,7 +272,9 @@ defmodule BlockScoutWeb.AddressInternalTransactionControllerTest do
             from_address: address,
             index: index,
             block_number: transaction_3.block_number,
-            transaction_index: transaction_3.index
+            transaction_index: transaction_3.index,
+            block_hash: b_block.hash,
+            block_index: index
           )
         end)
 
@@ -265,7 +287,9 @@ defmodule BlockScoutWeb.AddressInternalTransactionControllerTest do
           from_address: address,
           index: 11,
           block_number: transaction_3.block_number,
-          transaction_index: transaction_3.index
+          transaction_index: transaction_3.index,
+          block_hash: b_block.hash,
+          block_index: 11
         )
 
       conn =
@@ -304,7 +328,9 @@ defmodule BlockScoutWeb.AddressInternalTransactionControllerTest do
           from_address: address,
           index: index,
           block_number: transaction.block_number,
-          transaction_index: transaction.index
+          transaction_index: transaction.index,
+          block_hash: transaction.block_hash,
+          block_index: index
         )
       end)
 
@@ -335,7 +361,9 @@ defmodule BlockScoutWeb.AddressInternalTransactionControllerTest do
           :internal_transaction,
           transaction: transaction,
           from_address: address,
-          index: index
+          index: index,
+          block_hash: transaction.block_hash,
+          block_index: index
         )
       end)
 

apps/block_scout_web/test/block_scout_web/controllers/address_read_contract_controller_test.exs

@@ -21,13 +21,15 @@ defmodule BlockScoutWeb.AddressReadContractControllerTest do
     test "successfully renders the page when the address is a contract", %{conn: conn} do
       contract_address = insert(:contract_address)
 
-      transaction = insert(:transaction, from_address: contract_address)
+      transaction = insert(:transaction, from_address: contract_address) |> with_block()
 
       insert(
         :internal_transaction_create,
         index: 0,
         transaction: transaction,
-        created_contract_address: contract_address
+        created_contract_address: contract_address,
+        block_hash: transaction.block_hash,
+        block_index: 0
       )
 
       insert(:smart_contract, address_hash: contract_address.hash)
@@ -42,13 +44,15 @@ defmodule BlockScoutWeb.AddressReadContractControllerTest do
     test "returns not found for an unverified contract", %{conn: conn} do
       contract_address = insert(:contract_address)
 
-      transaction = insert(:transaction, from_address: contract_address)
+      transaction = insert(:transaction, from_address: contract_address) |> with_block()
 
       insert(
         :internal_transaction_create,
         index: 0,
         transaction: transaction,
-        created_contract_address: contract_address
+        created_contract_address: contract_address,
+        block_hash: transaction.block_hash,
+        block_index: 0
       )
 
       conn = get(conn, address_read_contract_path(BlockScoutWeb.Endpoint, :index, contract_address.hash))

apps/block_scout_web/test/block_scout_web/controllers/address_transaction_controller_test.exs

@@ -126,7 +126,9 @@ defmodule BlockScoutWeb.AddressTransactionControllerTest do
         index: 0,
         created_contract_address: address,
         to_address: nil,
-        transaction: transaction
+        transaction: transaction,
+        block_hash: block.hash,
+        block_index: 0
       )
 
       conn = get(conn, address_transaction_path(conn, :index, address), %{"type" => "JSON"})

apps/block_scout_web/test/block_scout_web/controllers/api/rpc/address_controller_test.exs

@@ -88,21 +88,24 @@ defmodule BlockScoutWeb.API.RPC.AddressControllerTest do
       mining_address =
         insert(:address,
           fetched_coin_balance: 0,
-          fetched_coin_balance_block_number: 2,
+          fetched_coin_balance_block_number: 102,
           inserted_at: Timex.shift(now, minutes: -10)
         )
 
       mining_address_hash = to_string(mining_address.hash)
       # we space these very far apart so that we know it will consider the 0th block stale (it calculates how far
       # back we'd need to go to get 24 hours in the past)
-      insert(:block, number: 0, timestamp: Timex.shift(now, hours: -50), miner: mining_address)
+      Enum.each(0..100, fn i ->
-      insert(:block, number: 1, timestamp: Timex.shift(now, hours: -25), miner: mining_address)
+        insert(:block, number: i, timestamp: Timex.shift(now, hours: -(102 - i) * 25), miner: mining_address)
+      end)
+
+      insert(:block, number: 101, timestamp: Timex.shift(now, hours: -25), miner: mining_address)
       AverageBlockTime.refresh()
 
       address =
         insert(:address,
           fetched_coin_balance: 100,
-          fetched_coin_balance_block_number: 0,
+          fetched_coin_balance_block_number: 100,
           inserted_at: Timex.shift(now, minutes: -5)
         )
 
@@ -112,7 +115,7 @@ defmodule BlockScoutWeb.API.RPC.AddressControllerTest do
                                                      %{
                                                        id: id,
                                                        method: "eth_getBalance",
-                                                       params: [^address_hash, "0x1"]
+                                                       params: [^address_hash, "0x65"]
                                                      }
                                                    ],
                                                    _options ->
@@ -148,7 +151,7 @@ defmodule BlockScoutWeb.API.RPC.AddressControllerTest do
 
       assert received_address.hash == address.hash
       assert received_address.fetched_coin_balance == expected_wei
-      assert received_address.fetched_coin_balance_block_number == 1
+      assert received_address.fetched_coin_balance_block_number == 101
     end
   end
 
@@ -1453,7 +1456,13 @@ defmodule BlockScoutWeb.API.RPC.AddressControllerTest do
 
       internal_transaction =
         :internal_transaction_create
-        |> insert(transaction: transaction, index: 0, from_address: address)
+        |> insert(
+          transaction: transaction,
+          index: 0,
+          from_address: address,
+          block_hash: transaction.block_hash,
+          block_index: 0
+        )
         |> with_contract_creation(contract_address)
 
       params = %{
@@ -1502,7 +1511,9 @@ defmodule BlockScoutWeb.API.RPC.AddressControllerTest do
         transaction: transaction,
         index: 0,
         type: :reward,
-        error: "some error"
+        error: "some error",
+        block_hash: transaction.block_hash,
+        block_index: 0
       ]
 
       insert(:internal_transaction_create, internal_transaction_details)
@@ -1532,7 +1543,12 @@ defmodule BlockScoutWeb.API.RPC.AddressControllerTest do
         |> with_block()
 
       for index <- 0..2 do
-        insert(:internal_transaction_create, transaction: transaction, index: index)
+        insert(:internal_transaction_create,
+          transaction: transaction,
+          index: index,
+          block_hash: transaction.block_hash,
+          block_index: index
+        )
       end
 
       params = %{
@@ -1606,7 +1622,13 @@ defmodule BlockScoutWeb.API.RPC.AddressControllerTest do
 
       internal_transaction =
         :internal_transaction_create
-        |> insert(transaction: transaction, index: 0, from_address: address)
+        |> insert(
+          transaction: transaction,
+          index: 0,
+          from_address: address,
+          block_hash: transaction.block_hash,
+          block_index: 0
+        )
         |> with_contract_creation(contract_address)
 
       params = %{
@@ -1658,7 +1680,9 @@ defmodule BlockScoutWeb.API.RPC.AddressControllerTest do
         transaction: transaction,
         index: 0,
         type: :reward,
-        error: "some error"
+        error: "some error",
+        block_hash: transaction.block_hash,
+        block_index: 0
       ]
 
       insert(:internal_transaction_create, internal_transaction_details)
@@ -1693,7 +1717,9 @@ defmodule BlockScoutWeb.API.RPC.AddressControllerTest do
         internal_transaction_details = %{
           from_address: address,
           transaction: transaction,
-          index: index
+          index: index,
+          block_hash: transaction.block_hash,
+          block_index: index
         }
 
         insert(:internal_transaction_create, internal_transaction_details)
@@ -1787,7 +1813,9 @@ defmodule BlockScoutWeb.API.RPC.AddressControllerTest do
         insert(:token_transfer, %{
           token_contract_address: token_address,
           token_id: 666,
-          transaction: transaction
+          transaction: transaction,
+          block: transaction.block,
+          block_number: transaction.block_number
         })
 
       {:ok, _} = Chain.token_from_address_hash(token_transfer.token_contract_address_hash)
@@ -1804,7 +1832,7 @@ defmodule BlockScoutWeb.API.RPC.AddressControllerTest do
                |> get("/api", params)
                |> json_response(200)
 
-      assert result["value"] == to_string(token_transfer.token_id)
+      assert result["tokenID"] == to_string(token_transfer.token_id)
       assert response["status"] == "1"
       assert response["message"] == "OK"
       assert :ok = ExJsonSchema.Validator.validate(tokentx_schema(), response)
@@ -1817,7 +1845,9 @@ defmodule BlockScoutWeb.API.RPC.AddressControllerTest do
         |> insert()
         |> with_block()
 
-      token_transfer = insert(:token_transfer, transaction: transaction)
+      token_transfer =
+        insert(:token_transfer, block: transaction.block, transaction: transaction, block_number: block.number)
+
       {:ok, token} = Chain.token_from_address_hash(token_transfer.token_contract_address_hash)
 
       params = %{
@@ -1894,8 +1924,20 @@ defmodule BlockScoutWeb.API.RPC.AddressControllerTest do
         |> insert()
         |> with_block()
 
-      insert(:token_transfer, from_address: address, transaction: transaction)
+      insert(:token_transfer,
-      insert(:token_transfer, from_address: address, token_contract_address: contract_address, transaction: transaction)
+        from_address: address,
+        transaction: transaction,
+        block: transaction.block,
+        block_number: transaction.block_number
+      )
+
+      insert(:token_transfer,
+        from_address: address,
+        token_contract_address: contract_address,
+        transaction: transaction,
+        block: transaction.block,
+        block_number: transaction.block_number
+      )
 
       params = %{
         "module" => "account",
@@ -2615,6 +2657,7 @@ defmodule BlockScoutWeb.API.RPC.AddressControllerTest do
           "logIndex" => %{"type" => "string"},
           "value" => %{"type" => "string"},
           "tokenName" => %{"type" => "string"},
+          "tokenID" => %{"type" => "string"},
           "tokenSymbol" => %{"type" => "string"},
           "tokenDecimal" => %{"type" => "string"},
           "transactionIndex" => %{"type" => "string"},

apps/block_scout_web/test/block_scout_web/controllers/api/rpc/contract_controller_test.exs

@@ -1,5 +1,6 @@
 defmodule BlockScoutWeb.API.RPC.ContractControllerTest do
   use BlockScoutWeb.ConnCase
+  alias Explorer.Chain.SmartContract
   alias Explorer.{Chain, Factory}
 
   describe "listcontracts" do
@@ -70,10 +71,7 @@ defmodule BlockScoutWeb.API.RPC.ContractControllerTest do
       assert response["result"] == [
                %{
                  "ABI" => "Contract source code not verified",
-                 "Address" => to_string(address.hash),
+                 "Address" => to_string(address.hash)
-                 "CompilerVersion" => "",
-                 "ContractName" => "",
-                 "OptimizationUsed" => ""
                }
              ]
 
@@ -95,10 +93,7 @@ defmodule BlockScoutWeb.API.RPC.ContractControllerTest do
       assert response["result"] == [
                %{
                  "ABI" => "Contract source code not verified",
-                 "Address" => to_string(address.hash),
+                 "Address" => to_string(address.hash)
-                 "CompilerVersion" => "",
-                 "ContractName" => "",
-                 "OptimizationUsed" => ""
                }
              ]
 
@@ -124,10 +119,7 @@ defmodule BlockScoutWeb.API.RPC.ContractControllerTest do
       assert response["result"] == [
                %{
                  "ABI" => "Contract source code not verified",
-                 "Address" => to_string(address.hash),
+                 "Address" => to_string(address.hash)
-                 "CompilerVersion" => "",
-                 "ContractName" => "",
-                 "OptimizationUsed" => ""
                }
              ]
 
@@ -174,10 +166,7 @@ defmodule BlockScoutWeb.API.RPC.ContractControllerTest do
       assert response["result"] == [
                %{
                  "ABI" => "Contract source code not verified",
-                 "Address" => to_string(decompiled_smart_contract.address_hash),
+                 "Address" => to_string(decompiled_smart_contract.address_hash)
-                 "CompilerVersion" => "",
-                 "ContractName" => "",
-                 "OptimizationUsed" => ""
                }
              ]
 
@@ -199,10 +188,7 @@ defmodule BlockScoutWeb.API.RPC.ContractControllerTest do
       assert response["result"] == [
                %{
                  "ABI" => "Contract source code not verified",
-                 "Address" => to_string(smart_contract.address_hash),
+                 "Address" => to_string(smart_contract.address_hash)
-                 "CompilerVersion" => "",
-                 "ContractName" => "",
-                 "OptimizationUsed" => ""
                }
              ]
 
@@ -225,10 +211,7 @@ defmodule BlockScoutWeb.API.RPC.ContractControllerTest do
 
       assert %{
                "ABI" => "Contract source code not verified",
-               "Address" => to_string(smart_contract.address_hash),
+               "Address" => to_string(smart_contract.address_hash)
-               "CompilerVersion" => "",
-               "ContractName" => "",
-               "OptimizationUsed" => ""
              } in response["result"]
 
       refute to_string(non_match.address_hash) in Enum.map(response["result"], &Map.get(&1, "Address"))
@@ -251,10 +234,7 @@ defmodule BlockScoutWeb.API.RPC.ContractControllerTest do
       assert response["result"] == [
                %{
                  "ABI" => "Contract source code not verified",
-                 "Address" => to_string(contract_address.hash),
+                 "Address" => to_string(contract_address.hash)
-                 "CompilerVersion" => "",
-                 "ContractName" => "",
-                 "OptimizationUsed" => ""
                }
              ]
 
@@ -281,10 +261,7 @@ defmodule BlockScoutWeb.API.RPC.ContractControllerTest do
       assert response["result"] == [
                %{
                  "ABI" => "Contract source code not verified",
-                 "Address" => to_string(contract_address.hash),
+                 "Address" => to_string(contract_address.hash)
-                 "CompilerVersion" => "",
-                 "ContractName" => "",
-                 "OptimizationUsed" => ""
                }
              ]
 
@@ -423,7 +400,11 @@ defmodule BlockScoutWeb.API.RPC.ContractControllerTest do
           "CompilerVersion" => "",
           "OptimizationUsed" => "",
           "DecompiledSourceCode" => "",
-          "DecompilerVersion" => ""
+          "DecompilerVersion" => "",
+          "ConstructorArguments" => "",
+          "EVMVersion" => "",
+          "ExternalLibraries" => "",
+          "OptimizationRuns" => ""
         }
       ]
 
@@ -439,7 +420,7 @@ defmodule BlockScoutWeb.API.RPC.ContractControllerTest do
     end
 
     test "with a verified contract address", %{conn: conn} do
-      contract = insert(:smart_contract, optimization: true)
+      contract = insert(:smart_contract, optimization: true, optimization_runs: 200, evm_version: "default")
 
       params = %{
         "module" => "contract",
@@ -456,12 +437,158 @@ defmodule BlockScoutWeb.API.RPC.ContractControllerTest do
           "ABI" => Jason.encode!(contract.abi),
           "ContractName" => contract.name,
           "CompilerVersion" => contract.compiler_version,
-          "DecompiledSourceCode" => "Contract source code not decompiled.",
           # The contract's optimization value is true, so the expected value
           # for `OptimizationUsed` is "1". If it was false, the expected value
           # would be "0".
-          "DecompilerVersion" => "",
+          "OptimizationUsed" => "true",
-          "OptimizationUsed" => "1"
+          "OptimizationRuns" => 200,
+          "EVMVersion" => "default"
+        }
+      ]
+
+      assert response =
+               conn
+               |> get("/api", params)
+               |> json_response(200)
+
+      assert response["result"] == expected_result
+      assert response["status"] == "1"
+      assert response["message"] == "OK"
+      assert :ok = ExJsonSchema.Validator.validate(getsourcecode_schema(), response)
+    end
+
+    test "with constructor arguments", %{conn: conn} do
+      contract =
+        insert(:smart_contract,
+          optimization: true,
+          optimization_runs: 200,
+          evm_version: "default",
+          constructor_arguments:
+            "00000000000000000000000008e7592ce0d7ebabf42844b62ee6a878d4e1913e000000000000000000000000e1b6037da5f1d756499e184ca15254a981c92546"
+        )
+
+      params = %{
+        "module" => "contract",
+        "action" => "getsourcecode",
+        "address" => to_string(contract.address_hash)
+      }
+
+      expected_result = [
+        %{
+          "Address" => to_string(contract.address_hash),
+          "SourceCode" =>
+            "/**\n* Submitted for verification at blockscout.com on #{contract.inserted_at}\n*/\n" <>
+              contract.contract_source_code,
+          "ABI" => Jason.encode!(contract.abi),
+          "ContractName" => contract.name,
+          "CompilerVersion" => contract.compiler_version,
+          "OptimizationUsed" => "true",
+          "OptimizationRuns" => 200,
+          "EVMVersion" => "default",
+          "ConstructorArguments" =>
+            "00000000000000000000000008e7592ce0d7ebabf42844b62ee6a878d4e1913e000000000000000000000000e1b6037da5f1d756499e184ca15254a981c92546"
+        }
+      ]
+
+      assert response =
+               conn
+               |> get("/api", params)
+               |> json_response(200)
+
+      assert response["result"] == expected_result
+      assert response["status"] == "1"
+      assert response["message"] == "OK"
+      assert :ok = ExJsonSchema.Validator.validate(getsourcecode_schema(), response)
+    end
+
+    test "with external library", %{conn: conn} do
+      smart_contract_bytecode =
+        "0x608060405234801561001057600080fd5b5060df8061001f6000396000f3006080604052600436106049576000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff16806360fe47b114604e5780636d4ce63c146078575b600080fd5b348015605957600080fd5b5060766004803603810190808035906020019092919050505060a0565b005b348015608357600080fd5b50608a60aa565b6040518082815260200191505060405180910390f35b8060008190555050565b600080549050905600a165627a7a7230582040d82a7379b1ee1632ad4d8a239954fd940277b25628ead95259a85c5eddb2120029"
+
+      created_contract_address =
+        insert(
+          :address,
+          hash: "0x0f95fa9bc0383e699325f2658d04e8d96d87b90c",
+          contract_code: smart_contract_bytecode
+        )
+
+      transaction =
+        :transaction
+        |> insert()
+        |> with_block()
+
+      insert(
+        :internal_transaction_create,
+        transaction: transaction,
+        index: 0,
+        created_contract_address: created_contract_address,
+        created_contract_code: smart_contract_bytecode,
+        block_number: transaction.block_number,
+        block_hash: transaction.block_hash,
+        block_index: 0,
+        transaction_index: transaction.index
+      )
+
+      valid_attrs = %{
+        address_hash: "0x0f95fa9bc0383e699325f2658d04e8d96d87b90c",
+        name: "Test",
+        compiler_version: "0.4.23",
+        contract_source_code:
+          "pragma solidity ^0.4.23; contract SimpleStorage {uint storedData; function set(uint x) public {storedData = x; } function get() public constant returns (uint) {return storedData; } }",
+        abi: [
+          %{
+            "constant" => false,
+            "inputs" => [%{"name" => "x", "type" => "uint256"}],
+            "name" => "set",
+            "outputs" => [],
+            "payable" => false,
+            "stateMutability" => "nonpayable",
+            "type" => "function"
+          },
+          %{
+            "constant" => true,
+            "inputs" => [],
+            "name" => "get",
+            "outputs" => [%{"name" => "", "type" => "uint256"}],
+            "payable" => false,
+            "stateMutability" => "view",
+            "type" => "function"
+          }
+        ],
+        optimization: true,
+        optimization_runs: 200,
+        evm_version: "default"
+      }
+
+      external_libraries = [
+        %SmartContract.ExternalLibrary{:address_hash => "0xb18aed9518d735482badb4e8b7fd8d2ba425ce95", :name => "Test"},
+        %SmartContract.ExternalLibrary{:address_hash => "0x283539e1b1daf24cdd58a3e934d55062ea663c3f", :name => "Test2"}
+      ]
+
+      {:ok, %SmartContract{} = contract} = Chain.create_smart_contract(valid_attrs, external_libraries)
+
+      params = %{
+        "module" => "contract",
+        "action" => "getsourcecode",
+        "address" => to_string(contract.address_hash)
+      }
+
+      expected_result = [
+        %{
+          "Address" => to_string(contract.address_hash),
+          "SourceCode" =>
+            "/**\n* Submitted for verification at blockscout.com on #{contract.inserted_at}\n*/\n" <>
+              contract.contract_source_code,
+          "ABI" => Jason.encode!(contract.abi),
+          "ContractName" => contract.name,
+          "CompilerVersion" => contract.compiler_version,
+          "OptimizationUsed" => "true",
+          "OptimizationRuns" => 200,
+          "EVMVersion" => "default",
+          "ExternalLibraries" => [
+            %{"name" => "Test", "address_hash" => "0xb18aed9518d735482badb4e8b7fd8d2ba425ce95"},
+            %{"name" => "Test2", "address_hash" => "0x283539e1b1daf24cdd58a3e934d55062ea663c3f"}
+          ]
         }
       ]
 
@@ -508,9 +635,8 @@ defmodule BlockScoutWeb.API.RPC.ContractControllerTest do
         "ABI" => Jason.encode!(contract_code_info.abi),
         "ContractName" => contract_code_info.name,
         "CompilerVersion" => contract_code_info.version,
-        "DecompiledSourceCode" => "Contract source code not decompiled.",
+        "OptimizationUsed" => "false",
-        "DecompilerVersion" => "",
+        "EVMVersion" => nil
-        "OptimizationUsed" => "0"
       }
 
       assert response["status"] == "1"
@@ -578,9 +704,9 @@ defmodule BlockScoutWeb.API.RPC.ContractControllerTest do
                  contract_source_code
 
       assert result["ContractName"] == name
-      assert result["DecompiledSourceCode"] == "Contract source code not decompiled."
+      assert result["DecompiledSourceCode"] == nil
-      assert result["DecompilerVersion"] == ""
+      assert result["DecompilerVersion"] == nil
-      assert result["OptimizationUsed"] == "1"
+      assert result["OptimizationUsed"] == "true"
       assert :ok = ExJsonSchema.Validator.validate(verify_schema(), response)
     end
   end

apps/block_scout_web/test/block_scout_web/controllers/api/rpc/eth_controller_test.exs

@@ -76,7 +76,7 @@ defmodule BlockScoutWeb.API.RPC.EthControllerTest do
       block = insert(:block, number: 0)
 
       transaction = insert(:transaction, from_address: address) |> with_block(block)
-      insert(:log, address: address, transaction: transaction, data: "0x010101")
+      insert(:log, block: block, address: address, transaction: transaction, data: "0x010101")
 
       params = params(api_params, [%{"address" => to_string(address.hash)}])
 
@@ -94,7 +94,7 @@ defmodule BlockScoutWeb.API.RPC.EthControllerTest do
       block = insert(:block, number: 0)
 
       transaction = insert(:transaction, from_address: address) |> with_block(block)
-      insert(:log, address: address, transaction: transaction, data: "0x010101", first_topic: "0x01")
+      insert(:log, block: block, address: address, transaction: transaction, data: "0x010101", first_topic: "0x01")
 
       params = params(api_params, [%{"address" => to_string(address.hash), "topics" => ["0x01"]}])
 
@@ -112,8 +112,8 @@ defmodule BlockScoutWeb.API.RPC.EthControllerTest do
       block = insert(:block, number: 0)
 
       transaction = insert(:transaction, from_address: address) |> with_block(block)
-      insert(:log, address: address, transaction: transaction, data: "0x010101", first_topic: "0x01")
+      insert(:log, address: address, block: block, transaction: transaction, data: "0x010101", first_topic: "0x01")
-      insert(:log, address: address, transaction: transaction, data: "0x020202", first_topic: "0x00")
+      insert(:log, address: address, block: block, transaction: transaction, data: "0x020202", first_topic: "0x00")
 
       params = params(api_params, [%{"address" => to_string(address.hash), "topics" => [["0x01", "0x00"]]}])
 
@@ -127,14 +127,15 @@ defmodule BlockScoutWeb.API.RPC.EthControllerTest do
 
     test "paginates logs", %{conn: conn, api_params: api_params} do
       contract_address = insert(:contract_address)
+      block = insert(:block)
 
       transaction =
         :transaction
         |> insert(to_address: contract_address)
-        |> with_block()
+        |> with_block(block)
 
       inserted_records =
-        insert_list(2000, :log, address: contract_address, transaction: transaction, first_topic: "0x01")
+        insert_list(2000, :log, block: block, address: contract_address, transaction: transaction, first_topic: "0x01")
 
       params = params(api_params, [%{"address" => to_string(contract_address), "topics" => [["0x01"]]}])
 
@@ -191,10 +192,11 @@ defmodule BlockScoutWeb.API.RPC.EthControllerTest do
         transaction: transaction,
         data: "0x010101",
         first_topic: "0x01",
-        second_topic: "0x02"
+        second_topic: "0x02",
+        block: block
       )
 
-      insert(:log, address: address, transaction: transaction, data: "0x020202", first_topic: "0x01")
+      insert(:log, block: block, address: address, transaction: transaction, data: "0x020202", first_topic: "0x01")
 
       params = params(api_params, [%{"address" => to_string(address.hash), "topics" => ["0x01", "0x02"]}])
 
@@ -219,7 +221,8 @@ defmodule BlockScoutWeb.API.RPC.EthControllerTest do
         transaction: transaction,
         data: "0x010101",
         first_topic: "0x01",
-        second_topic: "0x02"
+        second_topic: "0x02",
+        block: block
       )
 
       insert(:log,
@@ -227,7 +230,8 @@ defmodule BlockScoutWeb.API.RPC.EthControllerTest do
         transaction: transaction,
         data: "0x020202",
         first_topic: "0x01",
-        second_topic: "0x03"
+        second_topic: "0x03",
+        block: block
       )
 
       params = params(api_params, [%{"address" => to_string(address.hash), "topics" => ["0x01", ["0x02", "0x03"]]}])
@@ -341,11 +345,11 @@ defmodule BlockScoutWeb.API.RPC.EthControllerTest do
       transaction2 = insert(:transaction, from_address: address) |> with_block(block2)
       transaction3 = insert(:transaction, from_address: address) |> with_block(block3)
 
-      insert(:log, address: address, transaction: transaction1, data: "0x010101")
+      insert(:log, block: block1, address: address, transaction: transaction1, data: "0x010101")
 
-      insert(:log, address: address, transaction: transaction2, data: "0x020202")
+      insert(:log, block: block2, address: address, transaction: transaction2, data: "0x020202")
 
-      insert(:log, address: address, transaction: transaction3, data: "0x030303")
+      insert(:log, block: block3, address: address, transaction: transaction3, data: "0x030303")
 
       changeset = Ecto.Changeset.change(block3, %{consensus: false})
 

apps/block_scout_web/test/block_scout_web/controllers/api/rpc/stats_controller_test.exs

@@ -104,24 +104,41 @@ defmodule BlockScoutWeb.API.RPC.StatsControllerTest do
     end
   end
 
-  describe "ethsupply" do
+  # todo: Temporarily disable this test because of unstable work in CI
-    test "returns total supply from DB", %{conn: conn} do
+  # describe "ethsupply" do
-      params = %{
+  #   test "returns total supply from DB", %{conn: conn} do
-        "module" => "stats",
+  #     params = %{
-        "action" => "ethsupply"
+  #       "module" => "stats",
-      }
+  #       "action" => "ethsupply"
-
+  #     }
-      assert response =
+
-               conn
+  #     assert response =
-               |> get("/api", params)
+  #              conn
-               |> json_response(200)
+  #              |> get("/api", params)
-
+  #              |> json_response(200)
-      assert response["result"] == "6"
+
-      assert response["status"] == "1"
+  #     assert response["result"] == "0"
-      assert response["message"] == "OK"
+  #     assert response["status"] == "1"
-      assert :ok = ExJsonSchema.Validator.validate(ethsupply_schema(), response)
+  #     assert response["message"] == "OK"
-    end
+  #     assert :ok = ExJsonSchema.Validator.validate(ethsupply_schema(), response)
-  end
+  #   end
+  # end
+
+  # describe "coinsupply" do
+  #   test "returns total supply minus a burnt number from DB in coins denomination", %{conn: conn} do
+  #     params = %{
+  #       "module" => "stats",
+  #       "action" => "coinsupply"
+  #     }
+
+  #     assert response =
+  #              conn
+  #              |> get("/api", params)
+  #              |> json_response(200)
+
+  #     assert response == 0.0
+  #   end
+  # end
 
   describe "ethprice" do
     setup :set_mox_global

apps/block_scout_web/test/block_scout_web/controllers/api/rpc/transaction_controller_test.exs

@@ -243,8 +243,7 @@ defmodule BlockScoutWeb.API.RPC.TransactionControllerTest do
 
       transaction_details = [
         status: :error,
-        error: error,
+        error: error
-        internal_transactions_indexed_at: DateTime.utc_now()
       ]
 
       transaction =
@@ -256,7 +255,9 @@ defmodule BlockScoutWeb.API.RPC.TransactionControllerTest do
         transaction: transaction,
         index: 0,
         type: :reward,
-        error: error
+        error: error,
+        block_hash: transaction.block_hash,
+        block_index: 0
       ]
 
       insert(:internal_transaction, internal_transaction_details)
@@ -285,8 +286,7 @@ defmodule BlockScoutWeb.API.RPC.TransactionControllerTest do
     test "with a txhash with failed status but awaiting internal transactions", %{conn: conn} do
       transaction_details = [
         status: :error,
-        error: nil,
+        error: nil
-        internal_transactions_indexed_at: nil
       ]
 
       transaction =
@@ -411,7 +411,9 @@ defmodule BlockScoutWeb.API.RPC.TransactionControllerTest do
           address: address,
           transaction: transaction,
           first_topic: "first topic",
-          second_topic: "second topic"
+          second_topic: "second topic",
+          block: block,
+          block_number: block.number
         )
       end)
 
@@ -486,7 +488,9 @@ defmodule BlockScoutWeb.API.RPC.TransactionControllerTest do
           address: address,
           transaction: transaction,
           first_topic: "first topic",
-          second_topic: "second topic"
+          second_topic: "second topic",
+          block: block,
+          block_number: block.number
         )
 
       params = %{

apps/block_scout_web/test/block_scout_web/controllers/transaction_internal_transaction_controller_test.exs

@@ -44,14 +44,18 @@ defmodule BlockScoutWeb.TransactionInternalTransactionControllerTest do
         transaction: transaction,
         index: 0,
         block_number: transaction.block_number,
-        transaction_index: transaction.index
+        transaction_index: transaction.index,
+        block_hash: transaction.block_hash,
+        block_index: 0
       )
 
       insert(:internal_transaction,
         transaction: transaction,
         index: 1,
         transaction_index: transaction.index,
-        block_number: transaction.block_number
+        block_number: transaction.block_number,
+        block_hash: transaction.block_hash,
+        block_index: 1
       )
 
       path = transaction_internal_transaction_path(BlockScoutWeb.Endpoint, :index, transaction.hash)
@@ -90,7 +94,9 @@ defmodule BlockScoutWeb.TransactionInternalTransactionControllerTest do
           transaction: transaction,
           index: 0,
           block_number: transaction.block_number,
-          transaction_index: transaction.index
+          transaction_index: transaction.index,
+          block_hash: transaction.block_hash,
+          block_index: 0
         )
         |> with_contract_creation(contract_address)
 
@@ -118,7 +124,9 @@ defmodule BlockScoutWeb.TransactionInternalTransactionControllerTest do
           transaction: transaction,
           index: 0,
           block_number: transaction.block_number,
-          transaction_index: transaction.index
+          transaction_index: transaction.index,
+          block_hash: transaction.block_hash,
+          block_index: 0
         )
 
       second_page_indexes =
@@ -128,7 +136,9 @@ defmodule BlockScoutWeb.TransactionInternalTransactionControllerTest do
             transaction: transaction,
             index: index,
             block_number: transaction.block_number,
-            transaction_index: transaction.index
+            transaction_index: transaction.index,
+            block_hash: transaction.block_hash,
+            block_index: index
           )
         end)
         |> Enum.map(& &1.index)
@@ -161,7 +171,9 @@ defmodule BlockScoutWeb.TransactionInternalTransactionControllerTest do
           transaction: transaction,
           index: index,
           block_number: transaction.block_number,
-          transaction_index: transaction.index
+          transaction_index: transaction.index,
+          block_hash: transaction.block_hash,
+          block_index: index
         )
       end)
 
@@ -190,7 +202,9 @@ defmodule BlockScoutWeb.TransactionInternalTransactionControllerTest do
           transaction: transaction,
           index: index,
           block_number: transaction.block_number,
-          transaction_index: transaction.index
+          transaction_index: transaction.index,
+          block_hash: transaction.block_hash,
+          block_index: index
         )
       end)
 

apps/block_scout_web/test/block_scout_web/controllers/transaction_log_controller_test.exs

@@ -29,7 +29,13 @@ defmodule BlockScoutWeb.TransactionLogControllerTest do
         |> with_block()
 
       address = insert(:address)
-      insert(:log, address: address, transaction: transaction)
+
+      insert(:log,
+        address: address,
+        transaction: transaction,
+        block: transaction.block,
+        block_number: transaction.block_number
+      )
 
       conn = get(conn, transaction_log_path(conn, :index, transaction), %{type: "JSON"})
 
@@ -46,7 +52,13 @@ defmodule BlockScoutWeb.TransactionLogControllerTest do
         |> with_block()
 
       address = insert(:address)
-      insert(:log, address: address, transaction: transaction)
+
+      insert(:log,
+        address: address,
+        transaction: transaction,
+        block: transaction.block,
+        block_number: transaction.block_number
+      )
 
       conn = get(conn, transaction_log_path(conn, :index, transaction), %{type: "JSON"})
 
@@ -73,11 +85,24 @@ defmodule BlockScoutWeb.TransactionLogControllerTest do
         |> insert()
         |> with_block()
 
-      log = insert(:log, transaction: transaction, index: 1)
+      log =
+        insert(:log,
+          transaction: transaction,
+          index: 1,
+          block: transaction.block,
+          block_number: transaction.block_number
+        )
 
       second_page_indexes =
         2..51
-        |> Enum.map(fn index -> insert(:log, transaction: transaction, index: index) end)
+        |> Enum.map(fn index ->
+          insert(:log,
+            transaction: transaction,
+            index: index,
+            block: transaction.block,
+            block_number: transaction.block_number
+          )
+        end)
         |> Enum.map(& &1.index)
 
       conn =
@@ -98,7 +123,14 @@ defmodule BlockScoutWeb.TransactionLogControllerTest do
         |> with_block()
 
       1..60
-      |> Enum.map(fn index -> insert(:log, transaction: transaction, index: index) end)
+      |> Enum.map(fn index ->
+        insert(:log,
+          transaction: transaction,
+          index: index,
+          block: transaction.block,
+          block_number: transaction.block_number
+        )
+      end)
 
       conn = get(conn, transaction_log_path(conn, :index, transaction), %{type: "JSON"})
 

apps/block_scout_web/test/block_scout_web/controllers/transaction_token_transfer_controller_test.exs

@@ -46,11 +46,11 @@ defmodule BlockScoutWeb.TransactionTokenTransferControllerTest do
     end
 
     test "includes token transfers for the transaction", %{conn: conn} do
-      transaction = insert(:transaction)
+      transaction = insert(:transaction) |> with_block()
 
-      insert(:token_transfer, transaction: transaction)
+      insert(:token_transfer, transaction: transaction, block: transaction.block, block_number: transaction.block_number)
 
-      insert(:token_transfer, transaction: transaction)
+      insert(:token_transfer, transaction: transaction, block: transaction.block, block_number: transaction.block_number)
 
       path = transaction_token_transfer_path(BlockScoutWeb.Endpoint, :index, transaction.hash)
 
@@ -106,7 +106,9 @@ defmodule BlockScoutWeb.TransactionTokenTransferControllerTest do
         insert(
           :token_transfer,
           transaction: transaction,
-          log_index: log_index
+          log_index: log_index,
+          block: transaction.block,
+          block_number: transaction.block_number
         )
       end)
 
@@ -129,7 +131,9 @@ defmodule BlockScoutWeb.TransactionTokenTransferControllerTest do
         insert(
           :token_transfer,
           transaction: transaction,
-          log_index: log_index
+          log_index: log_index,
+          block_number: transaction.block_number,
+          block: transaction.block
         )
       end)
 

apps/block_scout_web/test/block_scout_web/features/viewing_addresses_test.exs

@@ -80,7 +80,7 @@ defmodule BlockScoutWeb.ViewingAddressesTest do
     test "see the contract creator and transaction links", %{session: session} do
       address = insert(:address)
       contract = insert(:contract_address)
-      transaction = insert(:transaction, from_address: address, created_contract_address: contract)
+      transaction = insert(:transaction, from_address: address, created_contract_address: contract) |> with_block()
 
       internal_transaction =
         insert(
@@ -88,7 +88,9 @@ defmodule BlockScoutWeb.ViewingAddressesTest do
           index: 1,
           transaction: transaction,
           from_address: address,
-          created_contract_address: contract
+          created_contract_address: contract,
+          block_hash: transaction.block_hash,
+          block_index: 1
         )
 
       address_hash = AddressView.trimmed_hash(address.hash)
@@ -102,7 +104,7 @@ defmodule BlockScoutWeb.ViewingAddressesTest do
     test "see the contract creator and transaction links even when the creator is another contract", %{session: session} do
       lincoln = insert(:address)
       contract = insert(:contract_address)
-      transaction = insert(:transaction)
+      transaction = insert(:transaction) |> with_block()
       another_contract = insert(:contract_address)
 
       insert(
@@ -112,7 +114,9 @@ defmodule BlockScoutWeb.ViewingAddressesTest do
         from_address: lincoln,
         to_address: contract,
         created_contract_address: contract,
-        type: :call
+        type: :call,
+        block_hash: transaction.block_hash,
+        block_index: 1
       )
 
       internal_transaction =
@@ -121,7 +125,9 @@ defmodule BlockScoutWeb.ViewingAddressesTest do
           index: 2,
           transaction: transaction,
           from_address: contract,
-          created_contract_address: another_contract
+          created_contract_address: another_contract,
+          block_hash: transaction.block_hash,
+          block_index: 2
         )
 
       contract_hash = AddressView.trimmed_hash(contract.hash)
@@ -208,7 +214,9 @@ defmodule BlockScoutWeb.ViewingAddressesTest do
           to_address: address,
           index: 1,
           block_number: 7000,
-          transaction_index: 1
+          transaction_index: 1,
+          block_hash: transaction.block_hash,
+          block_index: 1
         )
 
       insert(:internal_transaction,
@@ -216,7 +224,9 @@ defmodule BlockScoutWeb.ViewingAddressesTest do
         from_address: address,
         index: 2,
         block_number: 8000,
-        transaction_index: 2
+        transaction_index: 2,
+        block_hash: transaction.block_hash,
+        block_index: 2
       )
 
       {:ok, %{internal_transaction_lincoln_to_address: internal_transaction_lincoln_to_address}}
@@ -251,7 +261,9 @@ defmodule BlockScoutWeb.ViewingAddressesTest do
           index: 2,
           from_address: addresses.lincoln,
           block_number: transaction.block_number,
-          transaction_index: transaction.index
+          transaction_index: transaction.index,
+          block_hash: transaction.block_hash,
+          block_index: 2
         )
 
       Notifier.handle_event({:chain_event, :internal_transactions, :realtime, [internal_transaction]})

apps/block_scout_web/test/block_scout_web/features/viewing_app_test.exs

@@ -7,7 +7,7 @@ defmodule BlockScoutWeb.ViewingAppTest do
   alias BlockScoutWeb.Counters.BlocksIndexedCounter
   alias Explorer.Counters.AddressesCounter
   alias Explorer.{Repo}
-  alias Explorer.Chain.{Transaction}
+  alias Explorer.Chain.PendingBlockOperation
 
   setup do
     start_supervised!(AddressesCounter)
@@ -29,6 +29,8 @@ defmodule BlockScoutWeb.ViewingAppTest do
 
       assert Decimal.cmp(Explorer.Chain.indexed_ratio(), Decimal.from_float(0.5)) == :eq
 
+      insert(:pending_block_operation, block_hash: block.hash, fetch_internal_transactions: true)
+
       session
       |> AppPage.visit_page()
       |> assert_has(AppPage.indexed_status("50% Blocks Indexed"))
@@ -46,6 +48,8 @@ defmodule BlockScoutWeb.ViewingAppTest do
 
       assert Decimal.cmp(Explorer.Chain.indexed_ratio(), 1) == :eq
 
+      insert(:pending_block_operation, block_hash: block.hash, fetch_internal_transactions: true)
+
       session
       |> AppPage.visit_page()
       |> assert_has(AppPage.indexed_status("Indexing Tokens"))
@@ -65,6 +69,8 @@ defmodule BlockScoutWeb.ViewingAppTest do
 
       assert Decimal.cmp(Explorer.Chain.indexed_ratio(), Decimal.from_float(0.5)) == :eq
 
+      insert(:pending_block_operation, block_hash: block.hash, fetch_internal_transactions: true)
+
       session
       |> AppPage.visit_page()
       |> assert_has(AppPage.indexed_status("50% Blocks Indexed"))
@@ -90,6 +96,8 @@ defmodule BlockScoutWeb.ViewingAppTest do
 
       assert Decimal.cmp(Explorer.Chain.indexed_ratio(), Decimal.from_float(0.9)) == :eq
 
+      insert(:pending_block_operation, block_hash: block.hash, fetch_internal_transactions: true)
+
       session
       |> AppPage.visit_page()
       |> assert_has(AppPage.indexed_status("90% Blocks Indexed"))
@@ -111,6 +119,10 @@ defmodule BlockScoutWeb.ViewingAppTest do
       |> insert()
       |> with_block(block)
 
+      block_hash = block.hash
+
+      insert(:pending_block_operation, block_hash: block_hash, fetch_internal_transactions: true)
+
       BlocksIndexedCounter.calculate_blocks_indexed()
 
       assert Decimal.cmp(Explorer.Chain.indexed_ratio(), 1) == :eq
@@ -119,7 +131,10 @@ defmodule BlockScoutWeb.ViewingAppTest do
       |> AppPage.visit_page()
       |> assert_has(AppPage.indexed_status("Indexing Tokens"))
 
-      Repo.update_all(Transaction, set: [internal_transactions_indexed_at: DateTime.utc_now()])
+      Repo.update_all(
+        from(p in PendingBlockOperation, where: p.block_hash == ^block_hash),
+        set: [fetch_internal_transactions: false]
+      )
 
       BlocksIndexedCounter.calculate_blocks_indexed()
 

apps/block_scout_web/test/block_scout_web/features/viewing_blocks_test.exs

@@ -59,7 +59,12 @@ defmodule BlockScoutWeb.ViewingBlocksTest do
 
       internal_transaction =
         :internal_transaction_create
-        |> insert(transaction: transaction, index: 0)
+        |> insert(
+          transaction: transaction,
+          index: 0,
+          block_hash: transaction.block_hash,
+          block_index: 1
+        )
         |> with_contract_creation(contract_address)
 
       session

apps/block_scout_web/test/block_scout_web/features/viewing_transactions_test.exs

@@ -44,9 +44,15 @@ defmodule BlockScoutWeb.ViewingTransactionsTest do
       )
       |> with_block(block, gas_used: Decimal.new(1_230_000_000_000_123_000), status: :ok)
 
-    insert(:log, address: lincoln, index: 0, transaction: transaction)
+    insert(:log, address: lincoln, index: 0, transaction: transaction, block: block, block_number: block.number)
-
+
-    internal = insert(:internal_transaction, index: 0, transaction: transaction)
+    internal =
+      insert(:internal_transaction,
+        index: 0,
+        transaction: transaction,
+        block_hash: transaction.block_hash,
+        block_index: 0
+      )
 
     {:ok,
      %{
@@ -95,7 +101,7 @@ defmodule BlockScoutWeb.ViewingTransactionsTest do
         |> with_contract_creation(contract_address)
 
       :internal_transaction_create
-      |> insert(transaction: transaction, index: 0)
+      |> insert(transaction: transaction, index: 0, block_hash: transaction.block_hash, block_index: 0)
       |> with_contract_creation(contract_address)
 
       session

apps/block_scout_web/test/block_scout_web/schema/query/node_test.exs

@@ -58,9 +58,15 @@ defmodule BlockScoutWeb.Schema.Query.NodeTest do
     end
 
     test "with valid argument 'id' for an internal transaction", %{conn: conn} do
-      transaction = insert(:transaction)
+      transaction = insert(:transaction) |> with_block()
 
-      internal_transaction = insert(:internal_transaction, transaction: transaction, index: 0)
+      internal_transaction =
+        insert(:internal_transaction,
+          transaction: transaction,
+          index: 0,
+          block_hash: transaction.block_hash,
+          block_index: 0
+        )
 
       query = """
       query($id: ID!) {
@@ -96,9 +102,15 @@ defmodule BlockScoutWeb.Schema.Query.NodeTest do
     end
 
     test "with 'id' for non-existent internal transaction", %{conn: conn} do
-      transaction = build(:transaction)
+      transaction = insert(:transaction) |> with_block()
-
+
-      internal_transaction = build(:internal_transaction, transaction: transaction, index: 0)
+      internal_transaction =
+        build(:internal_transaction,
+          transaction: transaction,
+          index: 0,
+          block_hash: transaction.block_hash,
+          block_index: 0
+        )
 
       query = """
       query($id: ID!) {

apps/block_scout_web/test/block_scout_web/schema/query/transaction_test.exs

@@ -134,7 +134,9 @@ defmodule BlockScoutWeb.Schema.Query.TransactionTest do
         transaction: transaction,
         index: 0,
         from_address: address,
-        call_type: :call
+        call_type: :call,
+        block_hash: transaction.block_hash,
+        block_index: 0
       }
 
       internal_transaction =
@@ -259,11 +261,28 @@ defmodule BlockScoutWeb.Schema.Query.TransactionTest do
     end
 
     test "internal transactions are ordered by ascending index", %{conn: conn} do
-      transaction = insert(:transaction)
+      transaction = insert(:transaction) |> with_block()
+
+      insert(:internal_transaction,
+        transaction: transaction,
+        index: 2,
+        block_hash: transaction.block_hash,
+        block_index: 2
+      )
+
+      insert(:internal_transaction,
+        transaction: transaction,
+        index: 0,
+        block_hash: transaction.block_hash,
+        block_index: 0
+      )
 
-      insert(:internal_transaction, transaction: transaction, index: 2)
+      insert(:internal_transaction,
-      insert(:internal_transaction, transaction: transaction, index: 0)
+        transaction: transaction,
-      insert(:internal_transaction, transaction: transaction, index: 1)
+        index: 1,
+        block_hash: transaction.block_hash,
+        block_index: 1
+      )
 
       query = """
       query ($hash: FullHash!, $first: Int!) {
@@ -370,11 +389,28 @@ defmodule BlockScoutWeb.Schema.Query.TransactionTest do
       #
       # This test ensures support for a 'count' argument.
 
-      transaction = insert(:transaction)
+      transaction = insert(:transaction) |> with_block()
 
-      insert(:internal_transaction, transaction: transaction, index: 2)
+      insert(:internal_transaction,
-      insert(:internal_transaction, transaction: transaction, index: 0)
+        transaction: transaction,
-      insert(:internal_transaction, transaction: transaction, index: 1)
+        index: 2,
+        block_hash: transaction.block_hash,
+        block_index: 2
+      )
+
+      insert(:internal_transaction,
+        transaction: transaction,
+        index: 0,
+        block_hash: transaction.block_hash,
+        block_index: 0
+      )
+
+      insert(:internal_transaction,
+        transaction: transaction,
+        index: 1,
+        block_hash: transaction.block_hash,
+        block_index: 1
+      )
 
       query = """
       query ($hash: FullHash!, $last: Int!, $count: Int!) {
@@ -407,10 +443,15 @@ defmodule BlockScoutWeb.Schema.Query.TransactionTest do
     end
 
     test "pagination support with 'first' and 'after' arguments", %{conn: conn} do
-      transaction = insert(:transaction)
+      transaction = insert(:transaction) |> with_block()
 
       for index <- 0..5 do
-        insert(:internal_transaction_create, transaction: transaction, index: index)
+        insert(:internal_transaction_create,
+          transaction: transaction,
+          index: index,
+          block_hash: transaction.block_hash,
+          block_index: index
+        )
       end
 
       query1 = """

apps/block_scout_web/test/block_scout_web/views/address_view_test.exs

@@ -11,14 +11,16 @@ defmodule BlockScoutWeb.AddressViewTest do
     end
 
     test "for a pending internal transaction contract creation to address" do
-      transaction = insert(:transaction, to_address: nil)
+      transaction = insert(:transaction, to_address: nil) |> with_block()
 
       internal_transaction =
         insert(:internal_transaction,
           index: 1,
           transaction: transaction,
           to_address: nil,
-          created_contract_address_hash: nil
+          created_contract_address_hash: nil,
+          block_hash: transaction.block_hash,
+          block_index: 1
         )
 
       assert "Contract Address Pending" == AddressView.address_partial_selector(internal_transaction, :to, nil)

apps/block_scout_web/test/block_scout_web/views/transaction_view_test.exs

@@ -192,6 +192,8 @@ defmodule BlockScoutWeb.TransactionViewTest do
         |> insert()
         |> with_block(block, status: :error)
 
+      insert(:pending_block_operation, block_hash: block.hash, fetch_internal_transactions: true)
+
       status = TransactionView.transaction_status(transaction)
       assert TransactionView.formatted_status(status) == "Error: (Awaiting internal transactions for reason)"
     end
@@ -200,7 +202,7 @@ defmodule BlockScoutWeb.TransactionViewTest do
       transaction =
         :transaction
         |> insert()
-        |> with_block(status: :error, internal_transactions_indexed_at: DateTime.utc_now(), error: "Out of Gas")
+        |> with_block(status: :error, error: "Out of Gas")
 
       status = TransactionView.transaction_status(transaction)
       assert TransactionView.formatted_status(status) == "Error: Out of Gas"

apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/log.ex

@@ -39,6 +39,7 @@ defmodule EthereumJSONRPC.Log do
       ...> )
       %{
         address_hash: "0x8bf38d4764929064f2d4d3a56520a76ab3df415b",
+        block_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd",
         block_number: 37,
         data: "0x000000000000000000000000862d67cb0773ee3f8ce7ea89b328ffea861ab3ef",
         first_topic: "0x600bcf04a13e752d1e3670a5a9f1c21177ca2a93c6f5391d4f1298d098097c22",
@@ -47,6 +48,7 @@ defmodule EthereumJSONRPC.Log do
         second_topic: nil,
         third_topic: nil,
         transaction_hash: "0x53bd884872de3e488692881baeec262e7b95234d3965248c39fe992fffd433e5",
+        block_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd",
         type: "mined"
       }
 
@@ -69,7 +71,9 @@ defmodule EthereumJSONRPC.Log do
       ...> )
       %{
         address_hash: "0xda8b3276cde6d768a44b9dac659faa339a41ac55",
+        block_hash: "0x0b89f7f894f5d8ba941e16b61490e999a0fcaaf92dfcc70aee2ac5ddb5f243e1",
         block_number: 4448,
+        block_hash: "0x0b89f7f894f5d8ba941e16b61490e999a0fcaaf92dfcc70aee2ac5ddb5f243e1",
         data: "0x290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563",
         first_topic: "0xadc1e8a294f8415511303acc4a8c0c5906c7eb0bf2a71043d7f4b03b46a39130",
         fourth_topic: nil,
@@ -84,6 +88,7 @@ defmodule EthereumJSONRPC.Log do
         %{
           "address" => address_hash,
           "blockNumber" => block_number,
+          "blockHash" => block_hash,
           "data" => data,
           "logIndex" => index,
           "topics" => topics,
@@ -93,6 +98,7 @@ defmodule EthereumJSONRPC.Log do
     %{
       address_hash: address_hash,
       block_number: block_number,
+      block_hash: block_hash,
       data: data,
       index: index,
       transaction_hash: transaction_hash

apps/ethereum_jsonrpc/test/ethereum_jsonrpc/receipts_test.exs

@@ -34,6 +34,7 @@ defmodule EthereumJSONRPC.ReceiptsTest do
               cumulative_gas_used: 884_322,
               address_hash: "0x1e2fbe6be9eb39fc894d38be976111f332172d83",
               block_number: 3_560_000,
+              block_hash: nil,
               data:
                 "0x00000000000000000000000033066f6a8adf2d4f5db193524b6fbae062ec0d110000000000000000000000000000000000000000000000000000000000001030",
               index: 12,
@@ -48,6 +49,7 @@ defmodule EthereumJSONRPC.ReceiptsTest do
           EthereumJSONRPC.Parity ->
             %{
               created_contract_address_hash: nil,
+              block_hash: nil,
               cumulative_gas_used: 50450,
               gas_used: 50450,
               address_hash: "0x8bf38d4764929064f2d4d3a56520a76ab3df415b",
@@ -82,7 +84,9 @@ defmodule EthereumJSONRPC.ReceiptsTest do
                  "logs" => [
                    %{
                      "address" => address_hash,
+                     "blockHash" => "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd",
                      "blockNumber" => integer_to_quantity(block_number),
+                     "blockHash" => nil,
                      "data" => data,
                      "logIndex" => integer_to_quantity(index),
                      "topics" => [first_topic],

apps/explorer/config/config.exs

@@ -66,6 +66,11 @@ config :explorer, Explorer.Chain.Cache.AddressSum,
   ttl_check_interval: :timer.seconds(1),
   global_ttl: address_sum_global_ttl
 
+config :explorer, Explorer.Chain.Cache.AddressSumMinusBurnt,
+  enabled: true,
+  ttl_check_interval: :timer.seconds(1),
+  global_ttl: address_sum_global_ttl
+
 balances_update_interval =
   if System.get_env("ADDRESS_WITH_BALANCES_UPDATE_INTERVAL") do
     case Integer.parse(System.get_env("ADDRESS_WITH_BALANCES_UPDATE_INTERVAL")) do

apps/explorer/lib/explorer/application.ex

@@ -10,6 +10,7 @@ defmodule Explorer.Application do
   alias Explorer.Chain.Cache.{
     Accounts,
     AddressSum,
+    AddressSumMinusBurnt,
     BlockCount,
     BlockNumber,
     Blocks,
@@ -48,6 +49,7 @@ defmodule Explorer.Application do
       {Admin.Recovery, [[], [name: Admin.Recovery]]},
       TransactionCount,
       AddressSum,
+      AddressSumMinusBurnt,
       BlockCount,
       Blocks,
       NetVersion,

apps/explorer/lib/explorer/chain.ex

@@ -38,6 +38,7 @@ defmodule Explorer.Chain do
     Import,
     InternalTransaction,
     Log,
+    PendingBlockOperation,
     SmartContract,
     StakingPool,
     Token,
@@ -194,6 +195,7 @@ defmodule Explorer.Chain do
     paging_options = Keyword.get(options, :paging_options, @default_paging_options)
 
     InternalTransaction
+    |> InternalTransaction.where_nonpending_block()
     |> InternalTransaction.where_address_fields_match(hash, direction)
     |> InternalTransaction.where_is_different_from_parent_transaction()
     |> InternalTransaction.where_block_number_is_not_null()
@@ -406,9 +408,9 @@ defmodule Explorer.Chain do
 
     base_query =
       from(log in Log,
-        inner_join: transaction in assoc(log, :transaction),
+        inner_join: transaction in Transaction,
-        order_by: [desc: transaction.block_number, desc: transaction.index],
+        on: transaction.hash == log.transaction_hash,
-        preload: [:transaction, transaction: [to_address: :smart_contract]],
+        order_by: [desc: log.block_number, desc: log.index],
         where: transaction.block_number < ^block_number,
         or_where: transaction.block_number == ^block_number and transaction.index > ^transaction_index,
         or_where:
@@ -419,7 +421,19 @@ defmodule Explorer.Chain do
         select: log
       )
 
-    base_query
+    wrapped_query =
+      from(
+        log in subquery(base_query),
+        inner_join: transaction in Transaction,
+        preload: [:transaction, transaction: [to_address: :smart_contract]],
+        where:
+          log.block_hash == transaction.block_hash and
+            log.block_number == transaction.block_number and
+            log.transaction_hash == transaction.hash,
+        select: log
+      )
+
+    wrapped_query
     |> filter_topic(options)
     |> Repo.all()
     |> Enum.take(paging_options.page_size)
@@ -769,33 +783,25 @@ defmodule Explorer.Chain do
   end
 
   @doc """
-  Checks to see if the chain is down indexing based on the transaction from the oldest block having
+  Checks to see if the chain is down indexing based on the transaction from the
-  an `internal_transactions_indexed_at` date.
+  oldest block and the `fetch_internal_transactions` pending operation
   """
   @spec finished_indexing?() :: boolean()
   def finished_indexing? do
-    transaction_exists =
+    with {:transactions_exist, true} <- {:transactions_exist, Repo.exists?(Transaction)},
-      Transaction
+         min_block_number when not is_nil(min_block_number) <- Repo.aggregate(Transaction, :min, :block_number) do
-      |> limit(1)
+      query =
-      |> Repo.one()
+        from(
-
+          b in Block,
-    min_block_number_transaction = Repo.aggregate(Transaction, :min, :block_number)
+          join: pending_ops in assoc(b, :pending_operations),
+          where: pending_ops.fetch_internal_transactions,
+          where: b.consensus and b.number == ^min_block_number
+        )
 
-    if transaction_exists do
+      !Repo.exists?(query)
-      if min_block_number_transaction do
-        Transaction
-        |> where([t], t.block_number == ^min_block_number_transaction and is_nil(t.internal_transactions_indexed_at))
-        |> limit(1)
-        |> Repo.one()
-        |> case do
-          nil -> true
-          _ -> false
-        end
-      else
-        false
-      end
     else
-      true
+      {:transactions_exist, false} -> true
+      nil -> false
     end
   end
 
@@ -1373,6 +1379,20 @@ defmodule Explorer.Chain do
     Repo.one!(query)
   end
 
+  @spec fetch_sum_coin_total_supply_minus_burnt() :: non_neg_integer
+  def fetch_sum_coin_total_supply_minus_burnt do
+    {:ok, burn_address_hash} = string_to_address_hash("0x0000000000000000000000000000000000000000")
+
+    query =
+      from(
+        a0 in Address,
+        select: fragment("SUM(a0.fetched_coin_balance)"),
+        where: a0.hash != ^burn_address_hash
+      )
+
+    Repo.one!(query) || 0
+  end
+
   @spec fetch_sum_coin_total_supply() :: non_neg_integer
   def fetch_sum_coin_total_supply do
     query =
@@ -1387,11 +1407,8 @@ defmodule Explorer.Chain do
   @doc """
   The number of `t:Explorer.Chain.InternalTransaction.t/0`.
 
-      iex> transaction =
+      iex> transaction = :transaction |> insert() |> with_block()
-      ...>   :transaction |>
+      iex> insert(:internal_transaction, index: 0, transaction: transaction, block_hash: transaction.block_hash, block_index: 0)
-      ...>   insert() |>
-      ...>   with_block()
-      iex> insert(:internal_transaction, index: 0, transaction: transaction)
       iex> Explorer.Chain.internal_transaction_count()
       1
 
@@ -1402,7 +1419,7 @@ defmodule Explorer.Chain do
 
   """
   def internal_transaction_count do
-    Repo.one!(from(it in "internal_transactions", select: fragment("COUNT(*)")))
+    Repo.aggregate(InternalTransaction.where_nonpending_block(), :count, :transaction_hash)
   end
 
   @doc """
@@ -1678,18 +1695,20 @@ defmodule Explorer.Chain do
   end
 
   @doc """
-  Returns a stream of all blocks with unfetched internal transactions.
+  Returns a stream of all blocks with unfetched internal transactions, using
+  the `pending_block_operation` table.
 
   Only blocks with consensus are returned.
 
       iex> non_consensus = insert(:block, consensus: false)
+      iex> insert(:pending_block_operation, block: non_consensus, fetch_internal_transactions: true)
       iex> unfetched = insert(:block)
-      iex> fetched = insert(:block, internal_transactions_indexed_at: DateTime.utc_now())
+      iex> insert(:pending_block_operation, block: unfetched, fetch_internal_transactions: true)
-      iex> to_be_refetched = insert(:block, refetch_needed: true)
+      iex> fetched = insert(:block)
+      iex> insert(:pending_block_operation, block: fetched, fetch_internal_transactions: false)
       iex> {:ok, number_set} = Explorer.Chain.stream_blocks_with_unfetched_internal_transactions(
-      ...>   [:number],
       ...>   MapSet.new(),
-      ...>   fn %Explorer.Chain.Block{number: number}, acc ->
+      ...>   fn number, acc ->
       ...>     MapSet.put(acc, number)
       ...>   end
       ...> )
@@ -1699,112 +1718,55 @@ defmodule Explorer.Chain do
       true
       iex> fetched.hash in number_set
       false
-      iex> to_be_refetched.number in number_set
-      false
 
   """
   @spec stream_blocks_with_unfetched_internal_transactions(
-          fields :: [
-            :consensus
-            | :difficulty
-            | :gas_limit
-            | :gas_used
-            | :hash
-            | :miner
-            | :miner_hash
-            | :nonce
-            | :number
-            | :parent_hash
-            | :size
-            | :timestamp
-            | :total_difficulty
-            | :transactions
-            | :internal_transactions_indexed_at
-          ],
           initial :: accumulator,
           reducer :: (entry :: term(), accumulator -> accumulator)
         ) :: {:ok, accumulator}
         when accumulator: term()
-  def stream_blocks_with_unfetched_internal_transactions(fields, initial, reducer) when is_function(reducer, 2) do
+  def stream_blocks_with_unfetched_internal_transactions(initial, reducer) when is_function(reducer, 2) do
     query =
       from(
         b in Block,
+        join: pending_ops in assoc(b, :pending_operations),
+        where: pending_ops.fetch_internal_transactions,
         where: b.consensus,
-        where: is_nil(b.internal_transactions_indexed_at),
+        select: b.number
-        where: not b.refetch_needed,
-        select: ^fields
       )
 
     Repo.stream_reduce(query, initial, reducer)
   end
 
-  @doc """
+  def remove_nonconsensus_blocks_from_pending_ops(block_hashes) do
-  Returns a stream of all collated transactions with unfetched internal transactions.
+    query =
+      from(
+        po in PendingBlockOperation,
+        where: po.block_hash in ^block_hashes
+      )
 
-  Only transactions that have been collated into a block are returned; pending transactions not in a block are filtered
+    {_, _} = Repo.delete_all(query)
-  out.
 
-      iex> pending = insert(:transaction)
+    :ok
-      iex> unfetched_collated =
+  end
-      ...>   :transaction |>
-      ...>   insert() |>
-      ...>   with_block()
-      iex> fetched_collated =
-      ...>   :transaction |>
-      ...>   insert() |>
-      ...>   with_block(internal_transactions_indexed_at: DateTime.utc_now())
-      iex> {:ok, hash_set} = Explorer.Chain.stream_transactions_with_unfetched_internal_transactions(
-      ...>   [:hash],
-      ...>   MapSet.new(),
-      ...>   fn %Explorer.Chain.Transaction{hash: hash}, acc ->
-      ...>     MapSet.put(acc, hash)
-      ...>   end
-      ...> )
-      iex> pending.hash in hash_set
-      false
-      iex> unfetched_collated.hash in hash_set
-      true
-      iex> fetched_collated.hash in hash_set
-      false
 
-  """
+  def remove_nonconsensus_blocks_from_pending_ops do
-  @spec stream_transactions_with_unfetched_internal_transactions(
-          fields :: [
-            :block_hash
-            | :internal_transactions_indexed_at
-            | :from_address_hash
-            | :gas
-            | :gas_price
-            | :hash
-            | :index
-            | :input
-            | :nonce
-            | :r
-            | :s
-            | :to_address_hash
-            | :v
-            | :value
-          ],
-          initial :: accumulator,
-          reducer :: (entry :: term(), accumulator -> accumulator)
-        ) :: {:ok, accumulator}
-        when accumulator: term()
-  def stream_transactions_with_unfetched_internal_transactions(fields, initial, reducer) when is_function(reducer, 2) do
     query =
       from(
-        t in Transaction,
+        po in PendingBlockOperation,
-        # exclude pending transactions and replaced transactions
+        inner_join: block in Block,
-        where: not is_nil(t.block_hash) and is_nil(t.internal_transactions_indexed_at),
+        on: block.hash == po.block_hash,
-        select: ^fields
+        where: block.consensus == false
       )
 
-    Repo.stream_reduce(query, initial, reducer)
+    {_, _} = Repo.delete_all(query)
+
+    :ok
   end
 
   @spec stream_transactions_with_unfetched_created_contract_codes(
           fields :: [
             :block_hash
-            | :internal_transactions_indexed_at
             | :created_contract_code_indexed_at
             | :from_address_hash
             | :gas
@@ -1839,7 +1801,6 @@ defmodule Explorer.Chain do
   @spec stream_mined_transactions(
           fields :: [
             :block_hash
-            | :internal_transactions_indexed_at
             | :created_contract_code_indexed_at
             | :from_address_hash
             | :gas
@@ -1871,7 +1832,6 @@ defmodule Explorer.Chain do
   @spec stream_pending_transactions(
           fields :: [
             :block_hash
-            | :internal_transactions_indexed_at
             | :created_contract_code_indexed_at
             | :from_address_hash
             | :gas
@@ -2253,8 +2213,8 @@ defmodule Explorer.Chain do
   ## Options
 
     * `:necessity_by_association` - use to load `t:association/0` as `:required` or `:optional`.  If an association is
-      `:required`, and the `t:Explorer.Chain.InternalTransaction.t/0` has no associated record for that association,
+      `:required`, and the `t:Explorer.Chain.Transaction.t/0` has no associated record for that association,
-      then the `t:Explorer.Chain.InternalTransaction.t/0` will not be included in the list.
+      then the `t:Explorer.Chain.Transaction.t/0` will not be included in the list.
     * `:paging_options` - a `t:Explorer.PagingOptions.t/0` used to specify the `:page_size` and
       `:key` (a tuple of the lowest/oldest `{block_number, index}`) and. Results will be the transactions older than
       the `block_number` and `index` that are passed.
@@ -2308,8 +2268,8 @@ defmodule Explorer.Chain do
   ## Options
 
     * `:necessity_by_association` - use to load `t:association/0` as `:required` or `:optional`.  If an association is
-      `:required`, and the `t:Explorer.Chain.InternalTransaction.t/0` has no associated record for that association,
+      `:required`, and the `t:Explorer.Chain.Transaction.t/0` has no associated record for that association,
-      then the `t:Explorer.Chain.InternalTransaction.t/0` will not be included in the list.
+      then the `t:Explorer.Chain.Transaction.t/0` will not be included in the list.
     * `:paging_options` - a `t:Explorer.PagingOptions.t/0` used to specify the `:page_size` (defaults to
       `#{@default_paging_options.page_size}`) and `:key` (a tuple of the lowest/oldest `{inserted_at, hash}`) and.
       Results will be the transactions older than the `inserted_at` and `hash` that are passed.
@@ -2489,6 +2449,7 @@ defmodule Explorer.Chain do
     |> for_parent_transaction(hash)
     |> join_associations(necessity_by_association)
     |> where_transaction_has_multiple_internal_transactions()
+    |> InternalTransaction.where_nonpending_block()
     |> page_internal_transaction(paging_options)
     |> limit(^paging_options.page_size)
     |> order_by([internal_transaction], asc: internal_transaction.index)
@@ -2514,8 +2475,15 @@ defmodule Explorer.Chain do
     necessity_by_association = Keyword.get(options, :necessity_by_association, %{})
     paging_options = Keyword.get(options, :paging_options, @default_paging_options)
 
-    Log
+    log_with_transactions =
-    |> join(:inner, [log], transaction in assoc(log, :transaction))
+      from(log in Log,
+        inner_join: transaction in Transaction,
+        on:
+          transaction.block_hash == log.block_hash and transaction.block_number == log.block_number and
+            transaction.hash == log.transaction_hash
+      )
+
+    log_with_transactions
     |> where([_, transaction], transaction.hash == ^transaction_hash)
     |> page_logs(paging_options)
     |> limit(^paging_options.page_size)
@@ -2546,7 +2514,11 @@ defmodule Explorer.Chain do
 
     TokenTransfer
     |> join(:inner, [token_transfer], transaction in assoc(token_transfer, :transaction))
-    |> where([_, transaction], transaction.hash == ^transaction_hash)
+    |> where(
+      [token_transfer, transaction],
+      transaction.hash == ^transaction_hash and token_transfer.block_hash == transaction.block_hash and
+        token_transfer.block_number == transaction.block_number
+    )
     |> TokenTransfer.page_token_transfer(paging_options)
     |> limit(^paging_options.page_size)
     |> order_by([token_transfer], asc: token_transfer.inserted_at)
@@ -2581,7 +2553,7 @@ defmodule Explorer.Chain do
   def transaction_to_status(%Transaction{status: nil}), do: :awaiting_internal_transactions
   def transaction_to_status(%Transaction{status: :ok}), do: :success
 
-  def transaction_to_status(%Transaction{status: :error, internal_transactions_indexed_at: nil, error: nil}),
+  def transaction_to_status(%Transaction{status: :error, error: nil}),
     do: {:error, :awaiting_internal_transactions}
 
   def transaction_to_status(%Transaction{status: :error, error: error}) when is_binary(error), do: {:error, error}
@@ -2973,7 +2945,7 @@ defmodule Explorer.Chain do
   """
   @spec total_supply :: non_neg_integer() | nil
   def total_supply do
-    supply_module().total()
+    supply_module().total() || 0
   end
 
   @doc """
@@ -3018,21 +2990,33 @@ defmodule Explorer.Chain do
         ) :: {:ok, accumulator}
         when accumulator: term()
   def stream_unfetched_token_instances(initial, reducer) when is_function(reducer, 2) do
+    nft_tokens =
+      from(
+        token in Token,
+        where: token.type == ^"ERC-721",
+        select: token.contract_address_hash
+      )
+
     query =
       from(
         token_transfer in TokenTransfer,
-        inner_join: token in Token,
+        inner_join: token in subquery(nft_tokens),
         on: token.contract_address_hash == token_transfer.token_contract_address_hash,
         left_join: instance in Instance,
         on:
           token_transfer.token_id == instance.token_id and
             token_transfer.token_contract_address_hash == instance.token_contract_address_hash,
-        where: token.type == ^"ERC-721" and is_nil(instance.token_id) and not is_nil(token_transfer.token_id),
+        where: is_nil(instance.token_id) and not is_nil(token_transfer.token_id),
-        distinct: [token_transfer.token_contract_address_hash, token_transfer.token_id],
         select: %{contract_address_hash: token_transfer.token_contract_address_hash, token_id: token_transfer.token_id}
       )
 
-    Repo.stream_reduce(query, initial, reducer)
+    distinct_query =
+      from(
+        q in subquery(query),
+        distinct: [q.contract_address_hash, q.token_id]
+      )
+
+    Repo.stream_reduce(distinct_query, initial, reducer)
   end
 
   @doc """

apps/explorer/lib/explorer/chain/block.ex

@@ -7,10 +7,10 @@ defmodule Explorer.Chain.Block do
 
   use Explorer.Schema
 
-  alias Explorer.Chain.{Address, Gas, Hash, Transaction}
+  alias Explorer.Chain.{Address, Gas, Hash, PendingBlockOperation, Transaction}
   alias Explorer.Chain.Block.{Reward, SecondDegreeRelation}
 
-  @optional_attrs ~w(internal_transactions_indexed_at size refetch_needed total_difficulty difficulty)a
+  @optional_attrs ~w(size refetch_needed total_difficulty difficulty)a
 
   @required_attrs ~w(consensus gas_limit gas_used hash miner_hash nonce number parent_hash timestamp)a
 
@@ -46,7 +46,6 @@ defmodule Explorer.Chain.Block do
    * `timestamp` - When the block was collated
    * `total_difficulty` - the total `difficulty` of the chain until this block.
    * `transactions` - the `t:Explorer.Chain.Transaction.t/0` in this block.
-   * `internal_transactions_indexed_at` - when `internal_transactions` were fetched by `Indexer`.
   """
   @type t :: %__MODULE__{
           consensus: boolean(),
@@ -63,7 +62,6 @@ defmodule Explorer.Chain.Block do
           timestamp: DateTime.t(),
           total_difficulty: difficulty(),
           transactions: %Ecto.Association.NotLoaded{} | [Transaction.t()],
-          internal_transactions_indexed_at: DateTime.t(),
           refetch_needed: boolean()
         }
 
@@ -78,7 +76,6 @@ defmodule Explorer.Chain.Block do
     field(:size, :integer)
     field(:timestamp, :utc_datetime_usec)
     field(:total_difficulty, :decimal)
-    field(:internal_transactions_indexed_at, :utc_datetime_usec)
     field(:refetch_needed, :boolean)
 
     timestamps()
@@ -97,6 +94,8 @@ defmodule Explorer.Chain.Block do
     has_many(:transaction_forks, Transaction.Fork, foreign_key: :uncle_hash)
 
     has_many(:rewards, Reward, foreign_key: :block_hash)
+
+    has_one(:pending_operations, PendingBlockOperation, foreign_key: :block_hash)
   end
 
   def changeset(%__MODULE__{} = block, attrs) do
@@ -116,11 +115,23 @@ defmodule Explorer.Chain.Block do
   end
 
   def blocks_without_reward_query do
+    consensus_blocks_query =
+      from(
+        b in __MODULE__,
+        where: b.consensus == true
+      )
+
+    validator_rewards =
+      from(
+        r in Reward,
+        where: r.address_type == ^"validator"
+      )
+
     from(
-      b in __MODULE__,
+      b in subquery(consensus_blocks_query),
-      left_join: r in Reward,
+      left_join: r in subquery(validator_rewards),
       on: [block_hash: b.hash],
-      where: is_nil(r.block_hash) and b.consensus == true
+      where: is_nil(r.block_hash)
     )
   end
 

apps/explorer/lib/explorer/chain/cache/address_sum.ex

@@ -20,7 +20,7 @@ defmodule Explorer.Chain.Cache.AddressSum do
     # See next `handle_fallback` definition
     get_async_task()
 
-    {:return, nil}
+    {:return, Decimal.new(0)}
   end
 
   defp handle_fallback(:async_task) do

apps/explorer/lib/explorer/chain/cache/address_sum_minus_burnt.ex

@@ -0,0 +1,53 @@
+defmodule Explorer.Chain.Cache.AddressSumMinusBurnt do
+  @moduledoc """
+  Cache for address sum minus burnt number.
+  """
+
+  require Logger
+
+  use Explorer.Chain.MapCache,
+    name: :address_sum_minus_burnt,
+    key: :sum_minus_burnt,
+    key: :async_task,
+    ttl_check_interval: Application.get_env(:explorer, __MODULE__)[:ttl_check_interval],
+    global_ttl: Application.get_env(:explorer, __MODULE__)[:global_ttl],
+    callback: &async_task_on_deletion(&1)
+
+  alias Explorer.Chain
+
+  defp handle_fallback(:sum_minus_burnt) do
+    # This will get the task PID if one exists and launch a new task if not
+    # See next `handle_fallback` definition
+    get_async_task()
+
+    {:return, Decimal.new(0)}
+  end
+
+  defp handle_fallback(:async_task) do
+    # If this gets called it means an async task was requested, but none exists
+    # so a new one needs to be launched
+    {:ok, task} =
+      Task.start(fn ->
+        try do
+          result = Chain.fetch_sum_coin_total_supply_minus_burnt()
+
+          set_sum_minus_burnt(result)
+        rescue
+          e ->
+            Logger.debug([
+              "Coudn't update address sum test #{inspect(e)}"
+            ])
+        end
+
+        set_async_task(nil)
+      end)
+
+    {:update, task}
+  end
+
+  # By setting this as a `callback` an async task will be started each time the
+  # `sum_minus_burnt` expires (unless there is one already running)
+  defp async_task_on_deletion({:delete, _, :sum_minus_burnt}), do: get_async_task()
+
+  defp async_task_on_deletion(_data), do: nil
+end

apps/explorer/lib/explorer/chain/import.ex

@@ -12,7 +12,8 @@ defmodule Explorer.Chain.Import do
     Import.Stage.Addresses,
     Import.Stage.AddressReferencing,
     Import.Stage.BlockReferencing,
-    Import.Stage.BlockFollowing
+    Import.Stage.BlockFollowing,
+    Import.Stage.BlockPending
   ]
 
   # in order so that foreign keys are inserted before being referenced

apps/explorer/lib/explorer/chain/import/runner/blocks.ex

@@ -8,7 +8,7 @@ defmodule Explorer.Chain.Import.Runner.Blocks do
   import Ecto.Query, only: [from: 2, subquery: 1]
 
   alias Ecto.{Changeset, Multi, Repo}
-  alias Explorer.Chain.{Address, Block, Import, InternalTransaction, Log, TokenTransfer, Transaction}
+  alias Explorer.Chain.{Address, Block, Import, PendingBlockOperation, Transaction}
   alias Explorer.Chain.Block.Reward
   alias Explorer.Chain.Import.Runner
   alias Explorer.Chain.Import.Runner.Address.CurrentTokenBalances
@@ -56,6 +56,9 @@ defmodule Explorer.Chain.Import.Runner.Blocks do
       # Note, needs to be executed after `lose_consensus` for lock acquisition
       insert(repo, changes_list, insert_options)
     end)
+    |> Multi.run(:new_pending_operations, fn repo, %{lose_consensus: nonconsensus_hashes} ->
+      new_pending_operations(repo, nonconsensus_hashes, hashes, insert_options)
+    end)
     |> Multi.run(:uncle_fetched_block_second_degree_relations, fn repo, _ ->
       update_block_second_degree_relations(repo, hashes, %{
         timeout:
@@ -83,18 +86,9 @@ defmodule Explorer.Chain.Import.Runner.Blocks do
         transactions: transactions
       })
     end)
-    |> Multi.run(:remove_nonconsensus_logs, fn repo, %{derive_transaction_forks: transactions} ->
-      remove_nonconsensus_logs(repo, transactions, insert_options)
-    end)
-    |> Multi.run(:remove_nonconsensus_internal_transactions, fn repo, %{derive_transaction_forks: transactions} ->
-      remove_nonconsensus_internal_transactions(repo, transactions, insert_options)
-    end)
     |> Multi.run(:acquire_contract_address_tokens, fn repo, _ ->
       acquire_contract_address_tokens(repo, consensus_block_numbers)
     end)
-    |> Multi.run(:remove_nonconsensus_token_transfers, fn repo, %{derive_transaction_forks: transactions} ->
-      remove_nonconsensus_token_transfers(repo, transactions, insert_options)
-    end)
     |> Multi.run(:delete_address_token_balances, fn repo, _ ->
       delete_address_token_balances(repo, consensus_block_numbers, insert_options)
     end)
@@ -125,7 +119,8 @@ defmodule Explorer.Chain.Import.Runner.Blocks do
     query =
       from(address_current_token_balance in Address.CurrentTokenBalance,
         where: address_current_token_balance.block_number in ^consensus_block_numbers,
-        select: address_current_token_balance.token_contract_address_hash
+        select: address_current_token_balance.token_contract_address_hash,
+        distinct: address_current_token_balance.token_contract_address_hash
       )
 
     contract_address_hashes = repo.all(query)
@@ -160,7 +155,6 @@ defmodule Explorer.Chain.Import.Runner.Blocks do
             gas_used: nil,
             cumulative_gas_used: nil,
             index: nil,
-            internal_transactions_indexed_at: nil,
             status: nil,
             error: nil,
             updated_at: ^updated_at
@@ -251,7 +245,6 @@ defmodule Explorer.Chain.Import.Runner.Blocks do
           difficulty: fragment("EXCLUDED.difficulty"),
           gas_limit: fragment("EXCLUDED.gas_limit"),
           gas_used: fragment("EXCLUDED.gas_used"),
-          internal_transactions_indexed_at: fragment("EXCLUDED.internal_transactions_indexed_at"),
           miner_hash: fragment("EXCLUDED.miner_hash"),
           nonce: fragment("EXCLUDED.nonce"),
           number: fragment("EXCLUDED.number"),
@@ -270,8 +263,7 @@ defmodule Explorer.Chain.Import.Runner.Blocks do
           fragment("EXCLUDED.miner_hash <> ?", block.miner_hash) or fragment("EXCLUDED.nonce <> ?", block.nonce) or
           fragment("EXCLUDED.number <> ?", block.number) or fragment("EXCLUDED.parent_hash <> ?", block.parent_hash) or
           fragment("EXCLUDED.size <> ?", block.size) or fragment("EXCLUDED.timestamp <> ?", block.timestamp) or
-          fragment("EXCLUDED.total_difficulty <> ?", block.total_difficulty) or
+          fragment("EXCLUDED.total_difficulty <> ?", block.total_difficulty)
-          fragment("EXCLUDED.internal_transactions_indexed_at <> ?", block.internal_transactions_indexed_at)
     )
   end
 
@@ -317,92 +309,26 @@ defmodule Explorer.Chain.Import.Runner.Blocks do
       {:error, %{exception: postgrex_error, consensus_block_numbers: consensus_block_numbers}}
   end
 
-  defp remove_nonconsensus_token_transfers(repo, forked_transaction_hashes, %{timeout: timeout}) do
+  defp new_pending_operations(repo, nonconsensus_hashes, hashes, %{timeout: timeout, timestamps: timestamps}) do
-    ordered_token_transfers =
+    sorted_pending_ops =
-      from(
+      nonconsensus_hashes
-        token_transfer in TokenTransfer,
+      |> MapSet.new()
-        where: token_transfer.transaction_hash in ^forked_transaction_hashes,
+      |> MapSet.union(MapSet.new(hashes))
-        select: token_transfer.transaction_hash,
+      |> Enum.sort()
-        # Enforce TokenTransfer ShareLocks order (see docs: sharelocks.md)
+      |> Enum.map(fn hash ->
-        order_by: [
+        %{block_hash: hash, fetch_internal_transactions: true}
-          token_transfer.transaction_hash,
+      end)
-          token_transfer.log_index
-        ],
-        lock: "FOR UPDATE"
-      )
-
-    query =
-      from(token_transfer in TokenTransfer,
-        select: map(token_transfer, [:transaction_hash, :log_index]),
-        inner_join: ordered_token_transfer in subquery(ordered_token_transfers),
-        on: ordered_token_transfer.transaction_hash == token_transfer.transaction_hash
-      )
-
-    {_count, deleted_token_transfers} = repo.delete_all(query, timeout: timeout)
-
-    {:ok, deleted_token_transfers}
-  rescue
-    postgrex_error in Postgrex.Error ->
-      {:error, %{exception: postgrex_error, transactions: forked_transaction_hashes}}
-  end
-
-  defp remove_nonconsensus_internal_transactions(repo, forked_transaction_hashes, %{timeout: timeout}) do
-    query =
-      from(
-        internal_transaction in InternalTransaction,
-        where: internal_transaction.transaction_hash in ^forked_transaction_hashes,
-        select: %{transaction_hash: internal_transaction.transaction_hash},
-        # Enforce InternalTransaction ShareLocks order (see docs: sharelocks.md)
-        order_by: [
-          internal_transaction.transaction_hash,
-          internal_transaction.index
-        ],
-        lock: "FOR UPDATE"
-      )
-
-    delete_query =
-      from(
-        i in InternalTransaction,
-        join: s in subquery(query),
-        on: i.transaction_hash == s.transaction_hash,
-        select: map(i, [:transaction_hash, :index])
-      )
-
-    {_count, deleted_internal_transactions} = repo.delete_all(delete_query, timeout: timeout)
-
-    {:ok, deleted_internal_transactions}
-  rescue
-    postgrex_error in Postgrex.Error ->
-      {:error, %{exception: postgrex_error, transactions: forked_transaction_hashes}}
-  end
-
-  defp remove_nonconsensus_logs(repo, forked_transaction_hashes, %{timeout: timeout}) do
-    ordered_logs =
-      from(
-        log in Log,
-        where: log.transaction_hash in ^forked_transaction_hashes,
-        select: log.transaction_hash,
-        # Enforce Log ShareLocks order (see docs: sharelocks.md)
-        order_by: [
-          log.transaction_hash,
-          log.index
-        ],
-        lock: "FOR UPDATE"
-      )
-
-    query =
-      from(log in Log,
-        select: map(log, [:transaction_hash, :index]),
-        inner_join: ordered_log in subquery(ordered_logs),
-        on: ordered_log.transaction_hash == log.transaction_hash
-      )
-
-    {_count, deleted_logs} = repo.delete_all(query, timeout: timeout)
 
-    {:ok, deleted_logs}
+    Import.insert_changes_list(
-  rescue
+      repo,
-    postgrex_error in Postgrex.Error ->
+      sorted_pending_ops,
-      {:error, %{exception: postgrex_error, transactions: forked_transaction_hashes}}
+      conflict_target: :block_hash,
+      on_conflict: PendingBlockOperation.default_on_conflict(),
+      for: PendingBlockOperation,
+      returning: true,
+      timeout: timeout,
+      timestamps: timestamps
+    )
   end
 
   defp delete_address_token_balances(_, [], _), do: {:ok, []}

apps/explorer/lib/explorer/chain/import/runner/internal_transactions.ex

@@ -6,20 +6,19 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactions do
   require Ecto.Query
   require Logger
 
+  alias Ecto.Adapters.SQL
   alias Ecto.{Changeset, Multi, Repo}
-  alias Explorer.Chain.{Block, Hash, Import, InternalTransaction, Transaction}
+  alias Explorer.Chain.{Block, Hash, Import, InternalTransaction, PendingBlockOperation, Transaction}
   alias Explorer.Chain.Import.Runner
 
-  import Ecto.Query, only: [from: 2]
+  import Ecto.Query, only: [from: 2, or_where: 3]
 
   @behaviour Runner
 
   # milliseconds
   @timeout 60_000
 
-  @type imported :: [
+  @type imported :: [InternalTransaction.t()]
-          %{required(:index) => non_neg_integer(), required(:transaction_hash) => Hash.Full.t()}
-        ]
 
   @impl Runner
   def ecto_schema_module, do: InternalTransaction
@@ -48,54 +47,73 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactions do
 
     update_transactions_options = %{timeout: transactions_timeout, timestamps: timestamps}
 
+    # filter out params with just `block_number` (indicating blocks without internal transactions)
+    internal_transactions_params = Enum.filter(changes_list, &Map.has_key?(&1, :type))
+
     # Enforce ShareLocks tables order (see docs: sharelocks.md)
     multi
-    |> Multi.run(:acquire_transactions, fn repo, _ ->
+    |> Multi.run(:acquire_blocks, fn repo, _ ->
-      acquire_transactions(repo, changes_list)
+      acquire_blocks(repo, changes_list)
     end)
-    |> Multi.run(:internal_transactions, fn repo, %{acquire_transactions: transactions} ->
+    |> Multi.run(:acquire_pending_internal_txs, fn repo, %{acquire_blocks: block_hashes} ->
-      insert(repo, changes_list, transactions, insert_options)
+      acquire_pending_internal_txs(repo, block_hashes)
     end)
-    |> Multi.run(:internal_transactions_indexed_at_transactions, fn repo, %{acquire_transactions: transactions} ->
+    |> Multi.run(:acquire_transactions, fn repo, %{acquire_pending_internal_txs: pending_block_hashes} ->
-      update_transactions(repo, transactions, update_transactions_options)
+      acquire_transactions(repo, pending_block_hashes)
+    end)
+    |> Multi.run(:invalid_block_numbers, fn _, %{acquire_transactions: transactions} ->
+      invalid_block_numbers(transactions, internal_transactions_params)
+    end)
+    |> Multi.run(:valid_internal_transactions, fn _,
+                                                  %{
+                                                    acquire_transactions: transactions,
+                                                    invalid_block_numbers: invalid_block_numbers
+                                                  } ->
+      valid_internal_transactions(transactions, internal_transactions_params, invalid_block_numbers)
+    end)
+    |> Multi.run(:remove_left_over_internal_transactions, fn repo,
+                                                             %{valid_internal_transactions: valid_internal_transactions} ->
+      remove_left_over_internal_transactions(repo, valid_internal_transactions)
+    end)
+    |> Multi.run(:internal_transactions, fn repo, %{valid_internal_transactions: valid_internal_transactions} ->
+      insert(repo, valid_internal_transactions, insert_options)
+    end)
+    |> Multi.run(:update_transactions, fn repo, %{valid_internal_transactions: valid_internal_transactions} ->
+      update_transactions(repo, valid_internal_transactions, update_transactions_options)
+    end)
+    |> Multi.run(:remove_consensus_of_invalid_blocks, fn repo, %{invalid_block_numbers: invalid_block_numbers} ->
+      remove_consensus_of_invalid_blocks(repo, invalid_block_numbers)
+    end)
+    |> Multi.run(:update_pending_blocks_status, fn repo,
+                                                   %{
+                                                     acquire_pending_internal_txs: pending_block_hashes,
+                                                     remove_consensus_of_invalid_blocks: invalid_block_hashes
+                                                   } ->
+      update_pending_blocks_status(repo, pending_block_hashes, invalid_block_hashes)
     end)
-    |> Multi.run(
-      :remove_consensus_of_missing_transactions_blocks,
-      fn repo, %{internal_transactions: inserted} = results_map ->
-        # NOTE: for this to work it has to follow the runner `internal_transactions_indexed_at_blocks`
-        block_hashes = Map.get(results_map, :internal_transactions_indexed_at_blocks, [])
-        remove_consensus_of_missing_transactions_blocks(repo, block_hashes, changes_list, inserted)
-      end
-    )
   end
 
   @impl Runner
   def timeout, do: @timeout
 
-  @spec insert(Repo.t(), [map], [Transaction.t()], %{
+  @spec insert(Repo.t(), [map], %{
           optional(:on_conflict) => Runner.on_conflict(),
           required(:timeout) => timeout,
           required(:timestamps) => Import.timestamps()
         }) ::
           {:ok, [%{index: non_neg_integer, transaction_hash: Hash.t()}]}
           | {:error, [Changeset.t()]}
-  defp insert(repo, changes_list, transactions, %{timeout: timeout, timestamps: timestamps} = options)
+  defp insert(repo, valid_internal_transactions, %{timeout: timeout, timestamps: timestamps} = options)
-       when is_list(changes_list) do
+       when is_list(valid_internal_transactions) do
     on_conflict = Map.get_lazy(options, :on_conflict, &default_on_conflict/0)
 
-    transactions_map = Map.new(transactions, &{&1.hash, &1})
+    ordered_changes_list = Enum.sort_by(valid_internal_transactions, &{&1.transaction_hash, &1.index})
-
-    final_changes_list =
-      changes_list
-      # Enforce InternalTransaction ShareLocks order (see docs: sharelocks.md)
-      |> Enum.sort_by(&{&1.transaction_hash, &1.index})
-      |> reject_missing_transactions(transactions_map)
 
     {:ok, internal_transactions} =
       Import.insert_changes_list(
         repo,
-        final_changes_list,
+        ordered_changes_list,
-        conflict_target: [:transaction_hash, :index],
+        conflict_target: [:block_hash, :block_index],
         for: InternalTransaction,
         on_conflict: on_conflict,
         returning: true,
@@ -119,24 +137,28 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactions do
           from_address_hash: fragment("EXCLUDED.from_address_hash"),
           gas: fragment("EXCLUDED.gas"),
           gas_used: fragment("EXCLUDED.gas_used"),
-          # Don't update `index` as it is part of the composite primary key and used for the conflict target
+          index: fragment("EXCLUDED.index"),
           init: fragment("EXCLUDED.init"),
           input: fragment("EXCLUDED.input"),
           output: fragment("EXCLUDED.output"),
           to_address_hash: fragment("EXCLUDED.to_address_hash"),
           trace_address: fragment("EXCLUDED.trace_address"),
-          # Don't update `transaction_hash` as it is part of the composite primary key and used for the conflict target
+          transaction_hash: fragment("EXCLUDED.transaction_hash"),
           transaction_index: fragment("EXCLUDED.transaction_index"),
           type: fragment("EXCLUDED.type"),
           value: fragment("EXCLUDED.value"),
           inserted_at: fragment("LEAST(?, EXCLUDED.inserted_at)", internal_transaction.inserted_at),
           updated_at: fragment("GREATEST(?, EXCLUDED.updated_at)", internal_transaction.updated_at)
+          # Don't update `block_hash` as it is used for the conflict target
+          # Don't update `block_index` as it is used for the conflict target
         ]
       ],
       # `IS DISTINCT FROM` is used because it allows `NULL` to be equal to itself
       where:
         fragment(
-          "(EXCLUDED.call_type, EXCLUDED.created_contract_address_hash, EXCLUDED.created_contract_code, EXCLUDED.error, EXCLUDED.from_address_hash, EXCLUDED.gas, EXCLUDED.gas_used, EXCLUDED.init, EXCLUDED.input, EXCLUDED.output, EXCLUDED.to_address_hash, EXCLUDED.trace_address, EXCLUDED.transaction_index, EXCLUDED.type, EXCLUDED.value) IS DISTINCT FROM (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+          "(EXCLUDED.transaction_hash, EXCLUDED.index, EXCLUDED.call_type, EXCLUDED.created_contract_address_hash, EXCLUDED.created_contract_code, EXCLUDED.error, EXCLUDED.from_address_hash, EXCLUDED.gas, EXCLUDED.gas_used, EXCLUDED.init, EXCLUDED.input, EXCLUDED.output, EXCLUDED.to_address_hash, EXCLUDED.trace_address, EXCLUDED.transaction_index, EXCLUDED.type, EXCLUDED.value) IS DISTINCT FROM (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+          internal_transaction.transaction_hash,
+          internal_transaction.index,
           internal_transaction.call_type,
           internal_transaction.created_contract_address_hash,
           internal_transaction.created_contract_code,
@@ -156,18 +178,42 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactions do
     )
   end
 
-  defp acquire_transactions(repo, internal_transactions) do
+  defp acquire_blocks(repo, changes_list) do
-    transaction_hashes =
+    block_numbers = Enum.map(changes_list, & &1.block_number)
-      internal_transactions
-      |> MapSet.new(& &1.transaction_hash)
-      |> MapSet.to_list()
 
+    query =
+      from(
+        b in Block,
+        where: b.number in ^block_numbers and b.consensus,
+        select: b.hash,
+        # Enforce Block ShareLocks order (see docs: sharelocks.md)
+        order_by: [asc: b.hash],
+        lock: "FOR UPDATE"
+      )
+
+    {:ok, repo.all(query)}
+  end
+
+  defp acquire_pending_internal_txs(repo, block_hashes) do
+    query =
+      from(
+        pending_ops in PendingBlockOperation,
+        where: pending_ops.block_hash in ^block_hashes,
+        where: pending_ops.fetch_internal_transactions,
+        select: pending_ops.block_hash,
+        # Enforce PendingBlockOperation ShareLocks order (see docs: sharelocks.md)
+        order_by: [asc: pending_ops.block_hash],
+        lock: "FOR UPDATE"
+      )
+
+    {:ok, repo.all(query)}
+  end
+
+  defp acquire_transactions(repo, pending_block_hashes) do
     query =
       from(
         t in Transaction,
-        where: t.hash in ^transaction_hashes,
+        where: t.block_hash in ^pending_block_hashes,
-        # do not consider pending transactions
-        where: not is_nil(t.block_hash),
         select: map(t, [:hash, :block_hash, :block_number]),
         # Enforce Transaction ShareLocks order (see docs: sharelocks.md)
         order_by: t.hash,
@@ -177,22 +223,115 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactions do
     {:ok, repo.all(query)}
   end
 
-  defp update_transactions(repo, transactions, %{
+  defp invalid_block_numbers(transactions, internal_transactions_params) do
+    # Finds all mistmatches between transactions and internal transactions
+    # for a block number:
+    # - there are no internal txs for some transactions
+    # - there are no transactions for some internal transactions
+    # - there are internal txs with a different block number than their transactions
+    # Returns block numbers where any of these issues is found
+
+    required_tuples = MapSet.new(transactions, &{&1.hash, &1.block_number})
+
+    candidate_tuples = MapSet.new(internal_transactions_params, &{&1.transaction_hash, &1.block_number})
+
+    all_tuples = MapSet.union(required_tuples, candidate_tuples)
+
+    common_tuples = MapSet.intersection(required_tuples, candidate_tuples)
+
+    invalid_numbers =
+      all_tuples
+      |> MapSet.difference(common_tuples)
+      |> MapSet.new(fn {_hash, block_number} -> block_number end)
+      |> MapSet.to_list()
+
+    {:ok, invalid_numbers}
+  end
+
+  defp valid_internal_transactions(transactions, internal_transactions_params, invalid_block_numbers) do
+    blocks_map = Map.new(transactions, &{&1.block_number, &1.block_hash})
+
+    valid_internal_txs =
+      internal_transactions_params
+      |> Enum.group_by(& &1.block_number)
+      |> Map.drop(invalid_block_numbers)
+      |> Enum.flat_map(fn {block_number, entries} ->
+        block_hash = Map.fetch!(blocks_map, block_number)
+
+        entries
+        |> Enum.sort_by(&{&1.transaction_hash, &1.index})
+        |> Enum.with_index()
+        |> Enum.map(fn {entry, index} ->
+          entry
+          |> Map.put(:block_hash, block_hash)
+          |> Map.put(:block_index, index)
+        end)
+      end)
+
+    {:ok, valid_internal_txs}
+  end
+
+  def defer_internal_transactions_primary_key(repo) do
+    # Allows internal_transactions primary key to not be checked during the
+    # DB transactions and instead be checked only at the end of it.
+    # This allows us to use a more efficient upserting logic, while keeping the
+    # uniqueness valid.
+    SQL.query(repo, "SET CONSTRAINTS internal_transactions_pkey DEFERRED")
+  end
+
+  def remove_left_over_internal_transactions(repo, valid_internal_transactions) do
+    # Removes internal transactions that were part of a block before a refetch
+    # and have not been upserted with new ones (if any exist).
+
+    case valid_internal_transactions do
+      [] ->
+        {:ok, []}
+
+      _ ->
+        try do
+          delete_query_for_block_hash_block_index =
+            valid_internal_transactions
+            |> Enum.group_by(& &1.block_hash, & &1.block_index)
+            |> Enum.map(fn {block_hash, indexes} -> {block_hash, Enum.max(indexes)} end)
+            |> Enum.reduce(InternalTransaction, fn {block_hash, max_index}, acc ->
+              or_where(acc, [it], it.block_hash == ^block_hash and it.block_index > ^max_index)
+            end)
+
+          # removes old recoreds with the same primary key (transaction hash, transaction index)
+          delete_query =
+            valid_internal_transactions
+            |> Enum.map(fn params -> {params.transaction_hash, params.index} end)
+            |> Enum.reduce(delete_query_for_block_hash_block_index, fn {transaction_hash, index}, acc ->
+              or_where(acc, [it], it.transaction_hash == ^transaction_hash and it.index == ^index)
+            end)
+
+          # ShareLocks order already enforced by `acquire_pending_internal_txs` (see docs: sharelocks.md)
+          {count, result} = repo.delete_all(delete_query, [])
+
+          {:ok, {count, result}}
+        rescue
+          postgrex_error in Postgrex.Error -> {:error, %{exception: postgrex_error}}
+        end
+    end
+  end
+
+  defp update_transactions(repo, valid_internal_transactions, %{
          timeout: timeout,
          timestamps: timestamps
        })
-       when is_list(transactions) do
+       when is_list(valid_internal_transactions) do
-    transaction_hashes = Enum.map(transactions, & &1.hash)
+    transaction_hashes =
+      valid_internal_transactions
+      |> MapSet.new(& &1.transaction_hash)
+      |> MapSet.to_list()
 
     update_query =
       from(
         t in Transaction,
-        # pending transactions are already excluded by `acquire_transactions`
         where: t.hash in ^transaction_hashes,
         # ShareLocks order already enforced by `acquire_transactions` (see docs: sharelocks.md)
         update: [
           set: [
-            internal_transactions_indexed_at: ^timestamps.updated_at,
             created_contract_address_hash:
               fragment(
                 "(SELECT it.created_contract_address_hash FROM internal_transactions AS it WHERE it.transaction_hash = ? ORDER BY it.index ASC LIMIT 1)",
@@ -209,7 +348,8 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactions do
                 t.hash,
                 type(^:ok, t.status),
                 type(^:error, t.status)
-              )
+              ),
+            updated_at: ^timestamps.updated_at
           ]
         ]
       )
@@ -224,26 +364,14 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactions do
     end
   end
 
-  # If not using Parity this is not relevant
+  defp remove_consensus_of_invalid_blocks(repo, invalid_block_numbers) do
-  defp remove_consensus_of_missing_transactions_blocks(_, [], _, _), do: {:ok, []}
-
-  defp remove_consensus_of_missing_transactions_blocks(repo, block_hashes, changes_list, inserted) do
-    inserted_block_numbers = MapSet.new(inserted, & &1.block_number)
-
-    missing_transactions_block_numbers =
-      changes_list
-      |> MapSet.new(& &1.block_number)
-      |> MapSet.difference(inserted_block_numbers)
-      |> MapSet.to_list()
-
     update_query =
       from(
         b in Block,
-        where: b.number in ^missing_transactions_block_numbers,
+        where: b.number in ^invalid_block_numbers and b.consensus,
-        where: b.hash in ^block_hashes,
+        select: b.hash,
-        select: b.number,
+        # ShareLocks order already enforced by `acquire_blocks` (see docs: sharelocks.md)
-        # ShareLocks order already enforced by `internal_transactions_indexed_at_blocks` (see docs: sharelocks.md)
+        update: [set: [consensus: false]]
-        update: [set: [consensus: false, internal_transactions_indexed_at: nil]]
       )
 
     try do
@@ -252,24 +380,39 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactions do
       Logger.debug(fn ->
         [
           "consensus removed from blocks with numbers: ",
-          inspect(missing_transactions_block_numbers),
+          inspect(invalid_block_numbers),
-          " because of missing transactions"
+          " because of mismatching transactions"
         ]
       end)
 
       {:ok, result}
     rescue
       postgrex_error in Postgrex.Error ->
-        {:error, %{exception: postgrex_error, missing_transactions_block_numbers: missing_transactions_block_numbers}}
+        {:error, %{exception: postgrex_error, invalid_block_numbers: invalid_block_numbers}}
     end
   end
 
-  defp reject_missing_transactions(ordered_changes_list, transactions_map) do
+  def update_pending_blocks_status(repo, pending_hashes, invalid_block_hashes) do
-    Enum.reject(ordered_changes_list, fn %{transaction_hash: hash} ->
+    valid_block_hashes =
-      transactions_map
+      pending_hashes
-      |> Map.get(hash, %{})
+      |> MapSet.new()
-      |> Map.get(:block_hash)
+      |> MapSet.difference(MapSet.new(invalid_block_hashes))
-      |> is_nil()
+      |> MapSet.to_list()
-    end)
+
+    delete_query =
+      from(
+        pending_ops in PendingBlockOperation,
+        where: pending_ops.block_hash in ^valid_block_hashes
+      )
+
+    try do
+      # ShreLocks order already enforced by `acquire_pending_internal_txs` (see docs: sharelocks.md)
+      {_count, deleted} = repo.delete_all(delete_query, [])
+
+      {:ok, deleted}
+    rescue
+      postgrex_error in Postgrex.Error ->
+        {:error, %{exception: postgrex_error, pending_hashes: valid_block_hashes}}
+    end
   end
 end

apps/explorer/lib/explorer/chain/import/runner/internal_transactions_indexed_at_blocks.ex

@@ -1,82 +0,0 @@
-defmodule Explorer.Chain.Import.Runner.InternalTransactionsIndexedAtBlocks do
-  @moduledoc """
-  Bulk updates `internal_transactions_indexed_at` for provided blocks
-  """
-
-  require Ecto.Query
-
-  alias Ecto.Multi
-  alias Explorer.Chain.Block
-  alias Explorer.Chain.Import.Runner
-
-  import Ecto.Query, only: [from: 2]
-
-  @behaviour Runner
-
-  # milliseconds
-  @timeout 60_000
-
-  @type imported :: [%{number: Block.block_number()}]
-
-  @impl Runner
-  def ecto_schema_module, do: Block
-
-  @impl Runner
-  def option_key, do: :internal_transactions_indexed_at_blocks
-
-  @impl Runner
-  def imported_table_row do
-    %{
-      value_type: "[%{number: Explorer.Chain.Block.block_number()}]",
-      value_description: "List of block numbers to set `internal_transactions_indexed_at` field for"
-    }
-  end
-
-  @impl Runner
-  def run(multi, changes_list, %{timestamps: timestamps} = options) when is_map(options) do
-    transactions_timeout = options[Runner.Transactions.option_key()][:timeout] || Runner.Transactions.timeout()
-
-    update_transactions_options = %{timeout: transactions_timeout, timestamps: timestamps}
-
-    multi
-    |> Multi.run(:internal_transactions_indexed_at_blocks, fn repo, _ ->
-      update_blocks(repo, changes_list, update_transactions_options)
-    end)
-  end
-
-  @impl Runner
-  def timeout, do: @timeout
-
-  defp update_blocks(_repo, [], %{}), do: {:ok, []}
-
-  defp update_blocks(repo, changes_list, %{
-         timeout: timeout,
-         timestamps: timestamps
-       })
-       when is_list(changes_list) do
-    block_numbers = Enum.map(changes_list, fn %{number: number} -> number end)
-
-    query =
-      from(
-        b in Block,
-        where: b.number in ^block_numbers and b.consensus,
-        # Enforce Block ShareLocks order (see docs: sharelocks.md)
-        order_by: [asc: b.hash],
-        lock: "FOR UPDATE"
-      )
-
-    try do
-      {_, result} =
-        repo.update_all(
-          from(b in Block, join: s in subquery(query), on: b.hash == s.hash, select: b.hash),
-          [set: [internal_transactions_indexed_at: timestamps.updated_at]],
-          timeout: timeout
-        )
-
-      {:ok, result}
-    rescue
-      postgrex_error in Postgrex.Error ->
-        {:error, %{exception: postgrex_error, block_numbers: block_numbers}}
-    end
-  end
-end

apps/explorer/lib/explorer/chain/import/runner/logs.ex

@@ -59,13 +59,13 @@ defmodule Explorer.Chain.Import.Runner.Logs do
     on_conflict = Map.get_lazy(options, :on_conflict, &default_on_conflict/0)
 
     # Enforce Log ShareLocks order (see docs: sharelocks.md)
-    ordered_changes_list = Enum.sort_by(changes_list, &{&1.transaction_hash, &1.index})
+    ordered_changes_list = Enum.sort_by(changes_list, &{&1.transaction_hash, &1.block_hash, &1.index})
 
     {:ok, _} =
       Import.insert_changes_list(
         repo,
         ordered_changes_list,
-        conflict_target: [:transaction_hash, :index],
+        conflict_target: [:transaction_hash, :index, :block_hash],
         on_conflict: on_conflict,
         for: Log,
         returning: true,

apps/explorer/lib/explorer/chain/import/runner/token_transfers.ex

@@ -55,13 +55,13 @@ defmodule Explorer.Chain.Import.Runner.TokenTransfers do
     on_conflict = Map.get_lazy(options, :on_conflict, &default_on_conflict/0)
 
     # Enforce TokenTransfer ShareLocks order (see docs: sharelocks.md)
-    ordered_changes_list = Enum.sort_by(changes_list, &{&1.transaction_hash, &1.log_index})
+    ordered_changes_list = Enum.sort_by(changes_list, &{&1.transaction_hash, &1.block_hash, &1.log_index})
 
     {:ok, _} =
       Import.insert_changes_list(
         repo,
         ordered_changes_list,
-        conflict_target: [:transaction_hash, :log_index],
+        conflict_target: [:transaction_hash, :log_index, :block_hash],
         on_conflict: on_conflict,
         for: TokenTransfer,
         returning: true,

apps/explorer/lib/explorer/chain/import/runner/transactions.ex

@@ -8,7 +8,7 @@ defmodule Explorer.Chain.Import.Runner.Transactions do
   import Ecto.Query, only: [from: 2]
 
   alias Ecto.{Multi, Repo}
-  alias Explorer.Chain.{Block, Data, Hash, Import, Transaction}
+  alias Explorer.Chain.{Block, Hash, Import, Transaction}
   alias Explorer.Chain.Import.Runner.TokenTransfers
 
   @behaviour Import.Runner
@@ -72,18 +72,14 @@ defmodule Explorer.Chain.Import.Runner.Transactions do
          changes_list,
          %{
            timeout: timeout,
-           timestamps: %{inserted_at: inserted_at} = timestamps,
+           timestamps: timestamps
-           token_transfer_transaction_hash_set: token_transfer_transaction_hash_set
          } = options
        )
        when is_list(changes_list) do
     on_conflict = Map.get_lazy(options, :on_conflict, &default_on_conflict/0)
 
-    ordered_changes_list =
+    # Enforce Transaction ShareLocks order (see docs: sharelocks.md)
-      changes_list
+    ordered_changes_list = Enum.sort_by(changes_list, & &1.hash)
-      |> put_internal_transactions_indexed_at(inserted_at, token_transfer_transaction_hash_set)
-      # Enforce Transaction ShareLocks order (see docs: sharelocks.md)
-      |> Enum.sort_by(& &1.hash)
 
     Import.insert_changes_list(
       repo,
@@ -114,7 +110,6 @@ defmodule Explorer.Chain.Import.Runner.Transactions do
           gas_price: fragment("EXCLUDED.gas_price"),
           gas_used: fragment("EXCLUDED.gas_used"),
           index: fragment("EXCLUDED.index"),
-          internal_transactions_indexed_at: fragment("EXCLUDED.internal_transactions_indexed_at"),
           input: fragment("EXCLUDED.input"),
           nonce: fragment("EXCLUDED.nonce"),
           r: fragment("EXCLUDED.r"),
@@ -130,7 +125,7 @@ defmodule Explorer.Chain.Import.Runner.Transactions do
       ],
       where:
         fragment(
-          "(EXCLUDED.block_hash, EXCLUDED.block_number, EXCLUDED.created_contract_address_hash, EXCLUDED.created_contract_code_indexed_at, EXCLUDED.cumulative_gas_used, EXCLUDED.cumulative_gas_used, EXCLUDED.from_address_hash, EXCLUDED.gas, EXCLUDED.gas_price, EXCLUDED.gas_used, EXCLUDED.index, EXCLUDED.internal_transactions_indexed_at, EXCLUDED.input, EXCLUDED.nonce, EXCLUDED.r, EXCLUDED.s, EXCLUDED.status, EXCLUDED.to_address_hash, EXCLUDED.v, EXCLUDED.value) IS DISTINCT FROM (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+          "(EXCLUDED.block_hash, EXCLUDED.block_number, EXCLUDED.created_contract_address_hash, EXCLUDED.created_contract_code_indexed_at, EXCLUDED.cumulative_gas_used, EXCLUDED.cumulative_gas_used, EXCLUDED.from_address_hash, EXCLUDED.gas, EXCLUDED.gas_price, EXCLUDED.gas_used, EXCLUDED.index, EXCLUDED.input, EXCLUDED.nonce, EXCLUDED.r, EXCLUDED.s, EXCLUDED.status, EXCLUDED.to_address_hash, EXCLUDED.v, EXCLUDED.value) IS DISTINCT FROM (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
           transaction.block_hash,
           transaction.block_number,
           transaction.created_contract_address_hash,
@@ -142,7 +137,6 @@ defmodule Explorer.Chain.Import.Runner.Transactions do
           transaction.gas_price,
           transaction.gas_used,
           transaction.index,
-          transaction.internal_transactions_indexed_at,
           transaction.input,
           transaction.nonce,
           transaction.r,
@@ -155,46 +149,6 @@ defmodule Explorer.Chain.Import.Runner.Transactions do
     )
   end
 
-  defp put_internal_transactions_indexed_at(changes_list, timestamp, token_transfer_transaction_hash_set) do
-    if Application.get_env(:explorer, :index_internal_transactions_for_token_transfers) do
-      changes_list
-    else
-      do_put_internal_transactions_indexed_at(changes_list, timestamp, token_transfer_transaction_hash_set)
-    end
-  end
-
-  defp do_put_internal_transactions_indexed_at(changes_list, timestamp, token_transfer_transaction_hash_set)
-       when is_list(changes_list) do
-    Enum.map(changes_list, &put_internal_transactions_indexed_at(&1, timestamp, token_transfer_transaction_hash_set))
-  end
-
-  defp do_put_internal_transactions_indexed_at(%{hash: hash} = changes, timestamp, token_transfer_transaction_hash_set) do
-    token_transfer? = to_string(hash) in token_transfer_transaction_hash_set
-
-    if put_internal_transactions_indexed_at?(changes, token_transfer?) do
-      Map.put(changes, :internal_transactions_indexed_at, timestamp)
-    else
-      changes
-    end
-  end
-
-  # A post-Byzantium validated transaction will have a status and if it has no input, it is a value transfer only.
-  # Internal transactions are only needed when status is `:error` to set `error`.
-  defp put_internal_transactions_indexed_at?(%{status: :ok, input: %Data{bytes: <<>>}}, _), do: true
-
-  # A post-Byzantium validated transaction will have a status and if it transfers tokens, the token transfer is in the
-  # log and the internal transactions.
-  # `created_contract_address_hash` must be `nil` because if a contract is created the internal transactions are needed
-  # to get
-  defp put_internal_transactions_indexed_at?(%{status: :ok} = changes, true) do
-    case Map.fetch(changes, :created_contract_address_hash) do
-      {:ok, created_contract_address_hash} when not is_nil(created_contract_address_hash) -> false
-      :error -> true
-    end
-  end
-
-  defp put_internal_transactions_indexed_at?(_, _), do: false
-
   defp discard_blocks_for_recollated_transactions(repo, changes_list, %{
          timeout: timeout,
          timestamps: %{updated_at: updated_at}

apps/explorer/lib/explorer/chain/import/stage/block_following.ex

@@ -13,10 +13,8 @@ defmodule Explorer.Chain.Import.Stage.BlockFollowing do
   @impl Stage
   def runners,
     do: [
-      Runner.InternalTransactionsIndexedAtBlocks,
       Runner.Block.SecondDegreeRelations,
       Runner.Block.Rewards,
-      Runner.InternalTransactions,
       Runner.Address.CurrentTokenBalances
     ]
 

apps/explorer/lib/explorer/chain/import/stage/block_pending.ex

@@ -0,0 +1,27 @@
+defmodule Explorer.Chain.Import.Stage.BlockPending do
+  @moduledoc """
+  Imports any tables that uses `Explorer.Chain.PendingBlockOperation` to track
+  progress and cannot be imported at the same time as those imported by
+  `Explorer.Chain.Import.Stage.Addresses`,
+  `Explorer.Chain.Import.Stage.AddressReferencing` and
+  `Explorer.Chain.Import.Stage.BlockReferencing`
+  """
+
+  alias Explorer.Chain.Import.{Runner, Stage}
+
+  @behaviour Stage
+
+  @impl Stage
+  def runners,
+    do: [
+      Runner.InternalTransactions
+    ]
+
+  @impl Stage
+  def multis(runner_to_changes_list, options) do
+    {final_multi, final_remaining_runner_to_changes_list} =
+      Stage.single_multi(runners(), runner_to_changes_list, options)
+
+    {[final_multi], final_remaining_runner_to_changes_list}
+  end
+end

apps/explorer/lib/explorer/chain/internal_transaction.ex

@@ -3,7 +3,7 @@ defmodule Explorer.Chain.InternalTransaction do
 
   use Explorer.Schema
 
-  alias Explorer.Chain.{Address, Data, Gas, Hash, Transaction, Wei}
+  alias Explorer.Chain.{Address, Block, Data, Gas, Hash, PendingBlockOperation, Transaction, Wei}
   alias Explorer.Chain.InternalTransaction.{Action, CallType, Result, Type}
 
   @typedoc """
@@ -22,11 +22,15 @@ defmodule Explorer.Chain.InternalTransaction do
    * `to_address` - the sink of the `value`
    * `to_address_hash` - hash of the sink of the `value`
    * `trace_address` - list of traces
-   * `transaction` - transaction in which this transaction occurred
+   * `transaction` - transaction in which this internal transaction occurred
    * `transaction_hash` - foreign key for `transaction`
    * `transaction_index` - the `t:Explorer.Chain.Transaction.t/0` `index` of `transaction` in `block_number`.
    * `type` - type of internal transaction
    * `value` - value of transferred from `from_address` to `to_address`
+   * `block` - block in which this internal transaction occurred
+   * `block_hash` - foreign key for `block`
+   * `block_index` - the index of this internal transaction inside the `block`
+   * `pending_block` - `nil` if `block` has all its internal transactions fetched
   """
   @type t :: %__MODULE__{
           block_number: Explorer.Chain.Block.block_number() | nil,
@@ -50,7 +54,9 @@ defmodule Explorer.Chain.InternalTransaction do
           transaction: %Ecto.Association.NotLoaded{} | Transaction.t(),
           transaction_hash: Hash.t(),
           transaction_index: Transaction.transaction_index() | nil,
-          value: Wei.t()
+          value: Wei.t(),
+          block_hash: Hash.Full.t(),
+          block_index: non_neg_integer()
         }
 
   @primary_key false
@@ -69,6 +75,7 @@ defmodule Explorer.Chain.InternalTransaction do
     field(:value, Wei)
     field(:block_number, :integer)
     field(:transaction_index, :integer)
+    field(:block_index, :integer)
 
     timestamps()
 
@@ -102,6 +109,20 @@ defmodule Explorer.Chain.InternalTransaction do
       references: :hash,
       type: Hash.Full
     )
+
+    belongs_to(:block, Block,
+      foreign_key: :block_hash,
+      references: :hash,
+      type: Hash.Full
+    )
+
+    belongs_to(:pending_block, PendingBlockOperation,
+      foreign_key: :block_hash,
+      define_field: false,
+      references: :block_hash,
+      type: Hash.Full,
+      where: [fetch_internal_transactions: true]
+    )
   end
 
   @doc """
@@ -125,7 +146,9 @@ defmodule Explorer.Chain.InternalTransaction do
       ...>     transaction_hash: "0x3a3eb134e6792ce9403ea4188e5e79693de9e4c94e499db132be086400da79e6",
       ...>     type: "create",
       ...>     value: 0,
-      ...>     block_number: 35
+      ...>     block_number: 35,
+      ...>     block_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd",
+      ...>     block_index: 0
       ...>   }
       ...> )
       iex> changeset.valid?
@@ -165,6 +188,8 @@ defmodule Explorer.Chain.InternalTransaction do
       ...>     type: "create",
       ...>     value: 0,
       ...>     block_number: 35,
+      ...>     block_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd",
+      ...>     block_index: 0,
       ...>     transaction_index: 0
       ...>   }
       iex> )
@@ -182,6 +207,8 @@ defmodule Explorer.Chain.InternalTransaction do
       ...>   %Explorer.Chain.InternalTransaction{},
       ...>   %{
       ...>     block_number: 35,
+      ...>     block_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd",
+      ...>     block_index: 0,
       ...>     transaction_index: 0,
       ...>     transaction_hash: "0x3a3eb134e6792ce9403ea4188e5e79693de9e4c94e499db132be086400da79e6",
       ...>     index: 0,
@@ -206,6 +233,8 @@ defmodule Explorer.Chain.InternalTransaction do
       ...>   %Explorer.Chain.InternalTransaction{},
       ...>   %{
       ...>     block_number: 35,
+      ...>     block_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd",
+      ...>     block_index: 0,
       ...>     transaction_index: 0,
       ...>     transaction_hash: "0xcd7c15dbbc797722bef6e1d551edfd644fc7f4fb2ccd6a7947b2d1ade9ed140b",
       ...>     index: 0,
@@ -230,6 +259,8 @@ defmodule Explorer.Chain.InternalTransaction do
       ...>   %Explorer.Chain.InternalTransaction{},
       ...>   %{
       ...>     block_number: 35,
+      ...>     block_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd",
+      ...>     block_index: 0,
       ...>     transaction_index: 0,
       ...>     transaction_hash: "0xcd7c15dbbc797722bef6e1d551edfd644fc7f4fb2ccd6a7947b2d1ade9ed140b",
       ...>     index: 0,
@@ -260,6 +291,8 @@ defmodule Explorer.Chain.InternalTransaction do
       ...>   %Explorer.Chain.InternalTransaction{},
       ...>   %{
       ...>     block_number: 35,
+      ...>     block_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd",
+      ...>     block_index: 0,
       ...>     transaction_index: 0,
       ...>     transaction_hash: "0xcd7c15dbbc797722bef6e1d551edfd644fc7f4fb2ccd6a7947b2d1ade9ed140b",
       ...>     index: 0,
@@ -300,6 +333,8 @@ defmodule Explorer.Chain.InternalTransaction do
       ...>     type: "create",
       ...>     value: 0,
       ...>     block_number: 35,
+      ...>     block_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd",
+      ...>     block_index: 0,
       ...>     transaction_index: 0
       ...>   }
       iex> )
@@ -326,6 +361,8 @@ defmodule Explorer.Chain.InternalTransaction do
       ...>     type: "create",
       ...>     value: 0,
       ...>     block_number: 35,
+      ...>     block_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd",
+      ...>     block_index: 0,
       ...>     transaction_index: 0
       ...>   }
       ...> )
@@ -351,6 +388,8 @@ defmodule Explorer.Chain.InternalTransaction do
       ...>     type: "selfdestruct",
       ...>     value: 0,
       ...>     block_number: 35,
+      ...>     block_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd",
+      ...>     block_index: 0,
       ...>     transaction_index: 0
       ...>   }
       ...> )
@@ -362,9 +401,34 @@ defmodule Explorer.Chain.InternalTransaction do
     internal_transaction
     |> cast(attrs, ~w(type)a)
     |> validate_required(~w(type)a)
+    |> validate_block_required(attrs)
     |> type_changeset(attrs)
   end
 
+  @doc """
+  Accepts changes without `:type` but with `:block_number`, if `:type` is defined
+  works like `changeset`, except allowing `:block_hash` and `:block_index` to be undefined.
+
+  This is used because the `internal_transactions` runner can derive such values
+  on its own or use empty types to know that a block has no internal transactions.
+  """
+  def blockless_changeset(%__MODULE__{} = internal_transaction, attrs \\ %{}) do
+    changeset = cast(internal_transaction, attrs, ~w(type block_number)a)
+
+    if validate_required(changeset, ~w(type)a).valid? do
+      type_changeset(changeset, attrs)
+    else
+      validate_required(changeset, ~w(block_number)a)
+    end
+  end
+
+  defp validate_block_required(changeset, attrs) do
+    changeset
+    |> cast(attrs, ~w(block_hash block_index)a)
+    |> validate_required(~w(block_hash block_index)a)
+    |> foreign_key_constraint(:block_hash)
+  end
+
   defp type_changeset(changeset, attrs) do
     type = get_field(changeset, :type)
 
@@ -503,6 +567,16 @@ defmodule Explorer.Chain.InternalTransaction do
     where(query, [t], not is_nil(t.block_number))
   end
 
+  @doc """
+  Filters out internal_transactions of blocks that are flagged as needing fethching
+  of internal_transactions
+  """
+  def where_nonpending_block(query \\ nil) do
+    (query || __MODULE__)
+    |> join(:left, [it], pending in assoc(it, :pending_block), as: :pending)
+    |> where([it, pending: pending], is_nil(pending.block_hash))
+  end
+
   def internal_transactions_to_raw(internal_transactions) when is_list(internal_transactions) do
     internal_transactions
     |> Enum.map(&internal_transaction_to_raw/1)

apps/explorer/lib/explorer/chain/log.ex

@@ -6,14 +6,16 @@ defmodule Explorer.Chain.Log do
   require Logger
 
   alias ABI.{Event, FunctionSelector}
-  alias Explorer.Chain.{Address, ContractMethod, Data, Hash, Transaction}
+  alias Explorer.Chain.{Address, Block, ContractMethod, Data, Hash, Transaction}
   alias Explorer.Repo
 
-  @required_attrs ~w(address_hash data index transaction_hash)a
+  @required_attrs ~w(address_hash data block_hash index transaction_hash)a
-  @optional_attrs ~w(first_topic second_topic third_topic fourth_topic type)a
+  @optional_attrs ~w(first_topic second_topic third_topic fourth_topic type block_number)a
 
   @typedoc """
    * `address` - address of contract that generate the event
+   * `block_hash` - hash of the block
+   * `block_number` - The block number that the transfer took place.
    * `address_hash` - foreign key for `address`
    * `data` - non-indexed log parameters.
    * `first_topic` - `topics[0]`
@@ -28,6 +30,8 @@ defmodule Explorer.Chain.Log do
   @type t :: %__MODULE__{
           address: %Ecto.Association.NotLoaded{} | Address.t(),
           address_hash: Hash.Address.t(),
+          block_hash: Hash.Full.t(),
+          block_number: non_neg_integer() | nil,
           data: Data.t(),
           first_topic: String.t(),
           second_topic: String.t(),
@@ -48,6 +52,7 @@ defmodule Explorer.Chain.Log do
     field(:fourth_topic, :string)
     field(:index, :integer, primary_key: true)
     field(:type, :string)
+    field(:block_number, :integer)
 
     timestamps()
 
@@ -59,6 +64,13 @@ defmodule Explorer.Chain.Log do
       references: :hash,
       type: Hash.Full
     )
+
+    belongs_to(:block, Block,
+      foreign_key: :block_hash,
+      primary_key: true,
+      references: :hash,
+      type: Hash.Full
+    )
   end
 
   @doc """
@@ -69,6 +81,7 @@ defmodule Explorer.Chain.Log do
       ...>   %Explorer.Chain.Log{},
       ...>   %{
       ...>     address_hash: "0x8bf38d4764929064f2d4d3a56520a76ab3df415b",
+      ...>     block_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd",
       ...>     data: "0x000000000000000000000000862d67cb0773ee3f8ce7ea89b328ffea861ab3ef",
       ...>     first_topic: "0x600bcf04a13e752d1e3670a5a9f1c21177ca2a93c6f5391d4f1298d098097c22",
       ...>     fourth_topic: nil,

apps/explorer/lib/explorer/chain/pending_block_operation.ex

@@ -0,0 +1,87 @@
+defmodule Explorer.Chain.PendingBlockOperation do
+  @moduledoc """
+  Tracks a block that has pending operations.
+  """
+
+  use Explorer.Schema
+
+  alias Explorer.Chain.{Block, Hash}
+
+  @required_attrs ~w(block_hash fetch_internal_transactions)a
+
+  @typedoc """
+   * `block_hash` - the hash of the block that has pending operations.
+   * `fetch_internal_transactions` - if the block needs its internal transactions fetched (or not)
+  """
+  @type t :: %__MODULE__{
+          block_hash: Hash.Full.t(),
+          fetch_internal_transactions: boolean()
+        }
+
+  @primary_key false
+  schema "pending_block_operations" do
+    field(:fetch_internal_transactions, :boolean)
+
+    timestamps()
+
+    belongs_to(:block, Block, foreign_key: :block_hash, primary_key: true, references: :hash, type: Hash.Full)
+  end
+
+  def changeset(%__MODULE__{} = pending_ops, attrs) do
+    pending_ops
+    |> cast(attrs, @required_attrs)
+    |> validate_required(@required_attrs)
+    |> foreign_key_constraint(:block_hash)
+    |> unique_constraint(:block_hash, name: :pending_block_operations_pkey)
+  end
+
+  @doc """
+  Returns all pending block operations with the `block_hash` in the given list,
+  using "FOR UPDATE" to grab ShareLocks in order (see docs: sharelocks.md)
+  """
+  def fetch_and_lock_by_hashes(hashes) when is_list(hashes) do
+    from(
+      pending_ops in __MODULE__,
+      where: pending_ops.block_hash in ^hashes,
+      order_by: [asc: pending_ops.block_hash],
+      lock: "FOR UPDATE"
+    )
+  end
+
+  def block_hashes(filter \\ nil)
+
+  def block_hashes(filter) when is_nil(filter) do
+    from(
+      pending_ops in __MODULE__,
+      select: pending_ops.block_hash
+    )
+  end
+
+  def block_hashes(filters) when is_list(filters) do
+    true_filters = Keyword.new(filters, &{&1, true})
+
+    from(
+      pending_ops in __MODULE__,
+      where: ^true_filters,
+      select: pending_ops.block_hash
+    )
+  end
+
+  def block_hashes(filter), do: block_hashes([filter])
+
+  def default_on_conflict do
+    from(
+      pending_ops in __MODULE__,
+      update: [
+        set: [
+          fetch_internal_transactions:
+            pending_ops.fetch_internal_transactions or fragment("EXCLUDED.fetch_internal_transactions"),
+          # Don't update `block_hash` as it is used for the conflict target
+          inserted_at: pending_ops.inserted_at,
+          updated_at: fragment("EXCLUDED.updated_at")
+        ]
+      ],
+      where: fragment("EXCLUDED.fetch_internal_transactions <> ?", pending_ops.fetch_internal_transactions)
+    )
+  end
+end

apps/explorer/lib/explorer/chain/token_transfer.ex

@@ -27,7 +27,7 @@ defmodule Explorer.Chain.TokenTransfer do
   import Ecto.Changeset
   import Ecto.Query, only: [from: 2, limit: 2, where: 3]
 
-  alias Explorer.Chain.{Address, Hash, TokenTransfer, Transaction}
+  alias Explorer.Chain.{Address, Block, Hash, TokenTransfer, Transaction}
   alias Explorer.Chain.Token.Instance
   alias Explorer.{PagingOptions, Repo}
 
@@ -35,6 +35,7 @@ defmodule Explorer.Chain.TokenTransfer do
 
   @typedoc """
   * `:amount` - The token transferred amount
+  * `:block_hash` - hash of the block
   * `:block_number` - The block number that the transfer took place.
   * `:from_address` - The `t:Explorer.Chain.Address.t/0` that sent the tokens
   * `:from_address_hash` - Address hash foreign key
@@ -50,6 +51,7 @@ defmodule Explorer.Chain.TokenTransfer do
   @type t :: %TokenTransfer{
           amount: Decimal.t(),
           block_number: non_neg_integer() | nil,
+          block_hash: Hash.Full.t(),
           from_address: %Ecto.Association.NotLoaded{} | Address.t(),
           from_address_hash: Hash.Address.t(),
           to_address: %Ecto.Association.NotLoaded{} | Address.t(),
@@ -93,6 +95,13 @@ defmodule Explorer.Chain.TokenTransfer do
       type: Hash.Full
     )
 
+    belongs_to(:block, Block,
+      foreign_key: :block_hash,
+      primary_key: true,
+      references: :hash,
+      type: Hash.Full
+    )
+
     has_one(
       :instance,
       Instance,
@@ -105,7 +114,7 @@ defmodule Explorer.Chain.TokenTransfer do
     timestamps()
   end
 
-  @required_attrs ~w(block_number log_index from_address_hash to_address_hash token_contract_address_hash transaction_hash)a
+  @required_attrs ~w(block_number log_index from_address_hash to_address_hash token_contract_address_hash transaction_hash block_hash)a
   @optional_attrs ~w(amount token_id)a
 
   @doc false

apps/explorer/lib/explorer/chain/transaction.ex

@@ -29,7 +29,7 @@ defmodule Explorer.Chain.Transaction do
   alias Explorer.Repo
 
   @optional_attrs ~w(block_hash block_number created_contract_address_hash cumulative_gas_used earliest_processing_start
-                     error gas_used index internal_transactions_indexed_at created_contract_code_indexed_at status
+                     error gas_used index created_contract_code_indexed_at status
                      to_address_hash)a
 
   @required_attrs ~w(from_address_hash gas gas_price hash input nonce r s v value)a
@@ -105,8 +105,6 @@ defmodule Explorer.Chain.Transaction do
    * `input`- data sent along with the transaction
    * `internal_transactions` - transactions (value transfers) created while executing contract used for this
      transaction
-   * `internal_transactions_indexed_at` - when `internal_transactions` were fetched by `Indexer` or when they do not
-     need to be fetched at `inserted_at`.
    * `created_contract_code_indexed_at` - when created `address` code was fetched by `Indexer`
 
      | `status` | `contract_creation_address_hash` | `input`    | Token Transfer? | `internal_transactions_indexed_at`        | `internal_transactions` | Description                                                                                         |
@@ -152,7 +150,6 @@ defmodule Explorer.Chain.Transaction do
           index: transaction_index | nil,
           input: Data.t(),
           internal_transactions: %Ecto.Association.NotLoaded{} | [InternalTransaction.t()],
-          internal_transactions_indexed_at: DateTime.t(),
           logs: %Ecto.Association.NotLoaded{} | [Log.t()],
           nonce: non_neg_integer(),
           r: r(),
@@ -174,7 +171,6 @@ defmodule Explorer.Chain.Transaction do
              :gas_price,
              :gas_used,
              :index,
-             :internal_transactions_indexed_at,
              :created_contract_code_indexed_at,
              :input,
              :nonce,
@@ -195,7 +191,6 @@ defmodule Explorer.Chain.Transaction do
     field(:gas_price, Wei)
     field(:gas_used, :decimal)
     field(:index, :integer)
-    field(:internal_transactions_indexed_at, :utc_datetime_usec)
     field(:created_contract_code_indexed_at, :utc_datetime_usec)
     field(:input, Data)
     field(:nonce, :integer)

apps/explorer/lib/explorer/counters/average_block_time.ex

@@ -63,24 +63,25 @@ defmodule Explorer.Counters.AverageBlockTime do
 
   defp refresh_timestamps do
     timestamps_query =
-      from(block in Block,
-        limit: 100,
-        offset: 0,
-        order_by: [desc: block.number],
-        select: {block.number, block.timestamp}
-      )
-
-    query =
       if Application.get_env(:explorer, :include_uncles_in_average_block_time) do
-        timestamps_query
+        from(block in Block,
+          limit: 100,
+          offset: 100,
+          order_by: [desc: block.number],
+          select: {block.number, block.timestamp}
+        )
       else
-        from(block in timestamps_query,
+        from(block in Block,
-          where: block.consensus == true
+          limit: 100,
+          offset: 100,
+          order_by: [desc: block.number],
+          where: block.consensus == true,
+          select: {block.number, block.timestamp}
         )
       end
 
     timestamps =
-      query
+      timestamps_query
       |> Repo.all()
       |> Enum.sort_by(fn {_, timestamp} -> timestamp end, &>=/2)
       |> Enum.map(fn {number, timestamp} ->

apps/explorer/lib/explorer/etherscan.ex

@@ -8,7 +8,7 @@ defmodule Explorer.Etherscan do
   alias Explorer.Etherscan.Logs
   alias Explorer.{Chain, Repo}
   alias Explorer.Chain.Address.TokenBalance
-  alias Explorer.Chain.{Block, Hash, InternalTransaction, Transaction}
+  alias Explorer.Chain.{Block, Hash, InternalTransaction, TokenTransfer, Transaction}
 
   @default_options %{
     order_by_direction: :desc,
@@ -97,6 +97,7 @@ defmodule Explorer.Etherscan do
 
     query
     |> Chain.where_transaction_has_multiple_internal_transactions()
+    |> InternalTransaction.where_nonpending_block()
     |> Repo.all()
   end
 
@@ -140,6 +141,7 @@ defmodule Explorer.Etherscan do
     |> where_address_match(address_hash, options)
     |> where_start_block_match(options)
     |> where_end_block_match(options)
+    |> InternalTransaction.where_nonpending_block()
     |> Repo.all()
   end
 
@@ -318,7 +320,8 @@ defmodule Explorer.Etherscan do
     query =
       from(
         t in Transaction,
-        inner_join: tt in assoc(t, :token_transfers),
+        inner_join: tt in TokenTransfer,
+        on: tt.transaction_hash == t.hash and tt.block_number == t.block_number and tt.block_hash == t.block_hash,
         inner_join: tkn in assoc(tt, :token),
         inner_join: b in assoc(t, :block),
         where: tt.from_address_hash == ^address_hash,

apps/explorer/lib/explorer/etherscan/logs.ex

@@ -79,7 +79,7 @@ defmodule Explorer.Etherscan.Logs do
     logs_query = where_topic_match(Log, prepared_filter)
 
     internal_transaction_log_query =
-      from(internal_transaction in InternalTransaction,
+      from(internal_transaction in InternalTransaction.where_nonpending_block(),
         join: transaction in assoc(internal_transaction, :transaction),
         join: log in ^logs_query,
         on: log.transaction_hash == internal_transaction.transaction_hash,

apps/explorer/lib/explorer/graphql.ex

@@ -40,7 +40,7 @@ defmodule Explorer.GraphQL do
   """
   @spec get_internal_transaction(map()) :: {:ok, InternalTransaction.t()} | {:error, String.t()}
   def get_internal_transaction(%{transaction_hash: _, index: _} = clauses) do
-    if internal_transaction = Repo.get_by(InternalTransaction, clauses) do
+    if internal_transaction = Repo.get_by(InternalTransaction.where_nonpending_block(), clauses) do
       {:ok, internal_transaction}
     else
       {:error, "Internal transaction not found."}
@@ -65,7 +65,9 @@ defmodule Explorer.GraphQL do
         select: it
       )
 
-    Chain.where_transaction_has_multiple_internal_transactions(query)
+    query
+    |> InternalTransaction.where_nonpending_block()
+    |> Chain.where_transaction_has_multiple_internal_transactions()
   end
 
   @doc """

apps/explorer/lib/explorer/token/instance_metadata_retriever.ex

@@ -80,7 +80,7 @@ defmodule Explorer.Token.InstanceMetadataRetriever do
   def fetch_json(%{"tokenURI" => {:ok, [json]}}) do
     {:ok, json} = decode_json(json)
 
-    {:ok, %{metadata: json}}
+    check_type(json)
   rescue
     e ->
       Logger.debug(["Unknown metadata format #{inspect(json)}. error #{inspect(e)}"],
@@ -101,11 +101,7 @@ defmodule Explorer.Token.InstanceMetadataRetriever do
       {:ok, %Response{body: body, status_code: 200}} ->
         {:ok, json} = decode_json(body)
 
-        if is_map(json) do
+        check_type(json)
-          {:ok, %{metadata: json}}
-        else
-          {:error, :wrong_metadata_type}
-        end
 
       {:ok, %Response{body: body}} ->
         {:error, body}
@@ -131,4 +127,12 @@ defmodule Explorer.Token.InstanceMetadataRetriever do
       |> Jason.decode()
     end
   end
+
+  defp check_type(json) when is_map(json) do
+    {:ok, %{metadata: json}}
+  end
+
+  defp check_type(_) do
+    {:error, :wrong_metadata_type}
+  end
 end

apps/explorer/priv/repo/migrations/20191018120546_create_pending_block_operations.exs

@@ -0,0 +1,14 @@
+defmodule Explorer.Repo.Migrations.CreatePendingBlockOperations do
+  use Ecto.Migration
+
+  def change do
+    create table(:pending_block_operations, primary_key: false) do
+      add(:block_hash, references(:blocks, column: :hash, type: :bytea, on_delete: :delete_all),
+        null: false,
+        primary_key: true
+      )
+
+      timestamps(null: false, type: :utc_datetime_usec)
+    end
+  end
+end

apps/explorer/priv/repo/migrations/20191018140054_add_pending_internal_txs_operation.exs

@@ -0,0 +1,122 @@
+defmodule Explorer.Repo.Migrations.AddPendingInternalTxsOperation do
+  use Ecto.Migration
+
+  def change do
+    alter table(:pending_block_operations) do
+      add(:fetch_internal_transactions, :boolean, null: false)
+    end
+
+    execute("""
+    INSERT INTO pending_block_operations
+    (block_hash, inserted_at, updated_at, fetch_internal_transactions)
+    SELECT b.hash, now(), now(), TRUE FROM blocks b
+    WHERE b.internal_transactions_indexed_at IS NULL
+    AND EXISTS (
+      SELECT 1
+      FROM transactions t
+      WHERE b.hash = t.block_hash
+      AND t.internal_transactions_indexed_at IS NULL
+    );
+    """)
+
+    alter table(:blocks) do
+      remove(:internal_transactions_indexed_at)
+    end
+
+    alter table(:transactions) do
+      remove(:internal_transactions_indexed_at)
+    end
+
+    alter table(:internal_transactions) do
+      add(:block_hash, :bytea)
+      add(:block_index, :integer)
+    end
+
+    execute("""
+    UPDATE internal_transactions itx
+    SET block_hash = with_block.block_hash, block_index = with_block.block_index
+    FROM (
+      SELECT i.transaction_hash,
+      i.index,
+      t.block_hash,
+      row_number() OVER(
+        PARTITION BY t.block_hash
+        ORDER BY i.transaction_hash, i.index
+      ) - 1 AS block_index
+      FROM internal_transactions i
+      JOIN transactions t
+      ON t.hash = i.transaction_hash
+    ) AS with_block
+    WHERE itx.transaction_hash = with_block.transaction_hash
+    AND itx.index = with_block.index;
+    """)
+
+    execute("""
+    DELETE FROM internal_transactions WHERE block_hash IS NULL;
+    """)
+
+    execute("""
+    DO $$
+    DECLARE
+        duplicates_count INTEGER := 0;
+        blocks_scanned INTEGER := 0;
+        temprow RECORD;
+    BEGIN
+        FOR temprow IN
+            SELECT number, hash FROM blocks
+        LOOP
+            blocks_scanned := blocks_scanned + 1;
+            IF EXISTS (
+                SELECT 1 FROM transactions WHERE block_hash = temprow.hash
+            ) THEN
+                IF EXISTS (
+                    SELECT block_hash, block_index FROM internal_transactions
+                    WHERE block_hash = temprow.hash
+                    GROUP BY block_hash, block_index HAVING COUNT(*) > 1
+                ) THEN
+                    duplicates_count := duplicates_count + 1;
+                    RAISE NOTICE '% duplicates, blocks scanned %, block #%, block hash is %', duplicates_count, blocks_scanned, temprow.number , temprow.hash;
+
+                    IF NOT EXISTS (
+                        SELECT 1 FROM pending_block_operations
+                        WHERE block_hash = temprow.hash
+                    ) THEN
+                        INSERT INTO pending_block_operations
+                        (block_hash, inserted_at, updated_at, fetch_internal_transactions)
+                        SELECT b.hash, now(), now(), TRUE FROM blocks b
+                        WHERE b.hash = temprow.hash;
+                    END IF;
+
+                    DELETE FROM internal_transactions
+                    WHERE block_hash = temprow.hash;
+
+                    RAISE NOTICE 'DELETED';
+                END IF;
+            END IF;
+        END LOOP;
+        RAISE NOTICE 'SCRIPT FINISHED';
+    END $$;
+    """)
+
+    execute("""
+    ALTER table internal_transactions
+    DROP CONSTRAINT internal_transactions_pkey,
+    ADD PRIMARY KEY (block_hash, block_index);
+    """)
+
+    alter table(:internal_transactions) do
+      modify(:block_hash, references(:blocks, column: :hash, type: :bytea), null: false)
+      modify(:block_index, :integer, null: false)
+    end
+
+    drop(
+      index(
+        :internal_transactions,
+        [:transaction_hash, :index],
+        name: :internal_transactions_transaction_hash_index_index
+      )
+    )
+
+    create_if_not_exists(index(:internal_transactions, [:transaction_hash, :index]))
+  end
+end

apps/explorer/priv/repo/migrations/20191121064805_add_block_hash_and_block_index_to_logs.exs

@@ -0,0 +1,43 @@
+defmodule Explorer.Repo.Migrations.AddBlockHashAndBlockIndexToLogs do
+  use Ecto.Migration
+
+  def change do
+    alter table(:logs) do
+      add(:block_hash, :bytea)
+      add(:block_number, :integer)
+    end
+
+    execute("""
+    UPDATE logs log
+    SET block_hash = with_block.block_hash,
+    block_number = with_block.block_number
+    FROM (
+      SELECT l.transaction_hash,
+      t.block_hash,
+      t.block_number
+      FROM logs l
+      JOIN transactions t
+      ON t.hash = l.transaction_hash
+    ) AS with_block
+    WHERE log.transaction_hash = with_block.transaction_hash;
+    """)
+
+    execute("""
+    DELETE FROM logs WHERE block_hash IS NULL;
+    """)
+
+    alter table(:logs) do
+      modify(:block_hash, references(:blocks, column: :hash, type: :bytea), null: false)
+    end
+
+    execute("""
+    ALTER table logs
+    DROP CONSTRAINT logs_pkey,
+    ADD PRIMARY KEY (transaction_hash, block_hash, index);
+    """)
+
+    drop(unique_index(:logs, [:transaction_hash, :index]))
+
+    create_if_not_exists(index(:logs, [:transaction_hash, :index]))
+  end
+end

apps/explorer/priv/repo/migrations/20191122062035_add_block_hash_to_token_transfers.exs

@@ -0,0 +1,40 @@
+defmodule Explorer.Repo.Migrations.AddBlockHashToTokenTransfers do
+  use Ecto.Migration
+
+  def change do
+    alter table(:token_transfers) do
+      add(:block_hash, :bytea)
+    end
+
+    execute("""
+    UPDATE token_transfers token_transfer
+    SET block_hash = with_block.block_hash
+    FROM (
+      SELECT transfer.transaction_hash,
+      t.block_hash
+      FROM token_transfers transfer
+      JOIN transactions t
+      ON t.hash = transfer.transaction_hash
+    ) AS with_block
+    WHERE token_transfer.transaction_hash = with_block.transaction_hash;
+    """)
+
+    execute("""
+    DELETE FROM token_transfers WHERE block_hash IS NULL;
+    """)
+
+    alter table(:token_transfers) do
+      modify(:block_hash, references(:blocks, column: :hash, type: :bytea), null: false)
+    end
+
+    execute("""
+    ALTER table token_transfers
+    DROP CONSTRAINT token_transfers_pkey,
+    ADD PRIMARY KEY (transaction_hash, block_hash, log_index);
+    """)
+
+    drop(unique_index(:token_transfers, [:transaction_hash, :log_index]))
+
+    create_if_not_exists(index(:token_transfers, [:transaction_hash, :log_index]))
+  end
+end

apps/explorer/priv/repo/migrations/20191208135613_block_rewards_block_hash_partial_index.exs

@@ -0,0 +1,9 @@
+defmodule Explorer.Repo.Migrations.BlockRewardsBlockHashPartialIndex do
+  use Ecto.Migration
+
+  def change do
+    execute(
+      "CREATE INDEX IF NOT EXISTS block_rewards_block_hash_partial_index on block_rewards(block_hash) WHERE address_type='validator';"
+    )
+  end
+end

apps/explorer/priv/repo/migrations/20191218120138_logs_block_number_index_index.exs

@@ -0,0 +1,7 @@
+defmodule Explorer.Repo.Migrations.LogsBlockNumberIndexIndex do
+  use Ecto.Migration
+
+  def change do
+    create_if_not_exists(index(:logs, ["block_number DESC, index DESC"]))
+  end
+end

apps/explorer/priv/repo/migrations/20191220113006_pending_block_operations_block_hash_partial_index.exs

@@ -0,0 +1,9 @@
+defmodule Explorer.Repo.Migrations.PendingBlockOperationsBlockHashPartialIndex do
+  use Ecto.Migration
+
+  def change do
+    execute(
+      "CREATE INDEX pending_block_operations_block_hash_index_partial ON pending_block_operations(block_hash) WHERE fetch_internal_transactions=true;"
+    )
+  end
+end

apps/explorer/priv/repo/migrations/scripts/20181108205650_large_additional_internal_transaction_constraints.sql

@@ -6,6 +6,7 @@
 -- IMPORTANT NOTE: after making all the corrections needed the script will NOT
 -- run the constraint validations because this may be a very long and taxing
 -- operation. To validate the constraint one can run, after the script fininshed:
+-- UPDATE (2019-11-04): use pending_block_operations table instead of internal_transactions
 
 -- ALTER TABLE internal_transactions VALIDATE CONSTRAINT call_has_call_type;
 -- ALTER TABLE internal_transactions VALIDATE CONSTRAINT call_has_input;
@@ -14,56 +15,62 @@
 DO $$
 DECLARE
    batch_size  integer := 10000; -- HOW MANY ITEMS WILL BE UPDATED AT A TIME
-   last_transaction_hash bytea; -- WILL CHECK ONLY TRANSACTIONS FOLLOWING THIS HASH (DESC)
+   last_block_number integer; -- WILL CHECK ONLY TRANSACTIONS FOLLOWING THIS HASH (DESC)
    last_fetched_batch_size integer;
 BEGIN
   RAISE NOTICE 'STARTING SCRIPT';
-  CREATE TEMP TABLE transactions_with_deprecated_internal_transactions(hash bytea NOT NULL);
+  CREATE TEMP TABLE blocks_with_deprecated_internal_transactions(block_number integer NOT NULL);
 
   LOOP
     RAISE NOTICE 'Fetching new batch of % transactions to correct', batch_size;
 
-    INSERT INTO transactions_with_deprecated_internal_transactions
+    INSERT INTO blocks_with_deprecated_internal_transactions
-    SELECT DISTINCT transaction_hash
+    SELECT DISTINCT a.block_number
-    FROM internal_transactions
+    FROM (
-    WHERE
+      SELECT DISTINCT i.block_number, i.transaction_index
-      (last_transaction_hash IS NULL OR transaction_hash < last_transaction_hash) AND
+      FROM internal_transactions i
-      -- call_has_call_type CONSTRAINT
+      WHERE
-      ((type = 'call' AND call_type IS NULL) OR
+        i.block_number IS NOT NULL
-      -- call_has_input CONSTRAINT
+      AND
-      (type = 'call' AND input IS NULL) OR
+        (last_block_number IS NULL OR i.block_number < last_block_number) AND
-      -- create_has_init CONSTRAINT
+        -- call_has_call_type CONSTRAINT
-      (type = 'create' AND init is NULL))
+        ((i.type = 'call' AND i.call_type IS NULL) OR
-    ORDER BY transaction_hash DESC LIMIT batch_size;
+        -- call_has_input CONSTRAINT
+        (i.type = 'call' AND i.input IS NULL) OR
+        -- create_has_init CONSTRAINT
+        (i.type = 'create' AND i.init is NULL))
+      ORDER BY i.block_number DESC, i.transaction_index LIMIT batch_size
+    ) a;
 
-    SELECT INTO last_fetched_batch_size count(*) FROM transactions_with_deprecated_internal_transactions;
+    SELECT INTO last_fetched_batch_size count(block_number) FROM blocks_with_deprecated_internal_transactions;
 
     RAISE NOTICE 'Batch of % transactions was fetched, starting their deprecation', last_fetched_batch_size;
 
-    -- UPDATE TRANSACTIONS
+    INSERT INTO pending_block_operations (block_hash, inserted_at, updated_at, fetch_internal_transactions)
-    UPDATE transactions
+    SELECT b.hash, NOW(), NOW(), true
-    SET internal_transactions_indexed_at = NULL,
+    FROM blocks_with_deprecated_internal_transactions bd, blocks b
-        error = NULL
+    WHERE bd.block_number = b.number
-    FROM transactions_with_deprecated_internal_transactions
+    AND b.consensus = true
-    WHERE transactions.hash = transactions_with_deprecated_internal_transactions.hash;
+    ON CONFLICT (block_hash) 
+    DO NOTHING;
 
     -- REMOVE THE DEPRECATED internal_transactions
     DELETE FROM internal_transactions
-    USING transactions_with_deprecated_internal_transactions
+    USING blocks_with_deprecated_internal_transactions
-    WHERE internal_transactions.transaction_hash = transactions_with_deprecated_internal_transactions.hash;
+    WHERE internal_transactions.block_number = blocks_with_deprecated_internal_transactions.block_number;
 
     -- COMMIT THE BATCH UPDATES
     CHECKPOINT;
 
-    -- UPDATE last_transaction_hash TO KEEP TRACK OF ROWS ALREADY CHECKED
+    -- UPDATE last_block_number TO KEEP TRACK OF ROWS ALREADY CHECKED
-    SELECT INTO last_transaction_hash hash
+    SELECT INTO last_block_number block_number
-    FROM transactions_with_deprecated_internal_transactions
+    FROM blocks_with_deprecated_internal_transactions
-    ORDER BY hash ASC LIMIT 1;
+    ORDER BY block_number ASC LIMIT 1;
 
-    RAISE NOTICE 'Last batch completed, last transaction hash: %', last_transaction_hash;
+    RAISE NOTICE 'Last batch completed, last block number: %', last_block_number;
 
     -- CLEAR THE TEMP TABLE
-    DELETE FROM transactions_with_deprecated_internal_transactions;
+    DELETE FROM blocks_with_deprecated_internal_transactions;
 
     -- EXIT IF ALL internal_transactions HAVE BEEN CHECKED ALREADY
     EXIT WHEN last_fetched_batch_size != batch_size;
@@ -71,5 +78,5 @@ BEGIN
 
   RAISE NOTICE 'SCRIPT FINISHED, all affected transactions have been deprecated';
 
-  DROP TABLE transactions_with_deprecated_internal_transactions;
+  DROP TABLE blocks_with_deprecated_internal_transactions;
 END $$;

apps/explorer/test/explorer/chain/cache/address_sum_minus_burnt.exs

@@ -0,0 +1,58 @@
+defmodule Explorer.Chain.Cache.AddressSumMinusBurntTest do
+  use Explorer.DataCase
+
+  alias Explorer.Chain.Cache.AddressSumMinusBurnt
+
+  setup do
+    Supervisor.terminate_child(Explorer.Supervisor, AddressSumMinusBurnt.child_id())
+    Supervisor.restart_child(Explorer.Supervisor, AddressSumMinusBurnt.child_id())
+    :ok
+  end
+
+  test "returns default address sum" do
+    result = AddressSumMinusBurnt.get_sum_minus_burnt()
+
+    assert result == Decimal.new(0)
+  end
+
+  test "updates cache if initial value is zero" do
+    insert(:address, fetched_coin_balance: 1)
+    insert(:address, fetched_coin_balance: 2)
+    insert(:address, fetched_coin_balance: 3)
+    insert(:address, hash: "0x0000000000000000000000000000000000000000", fetched_coin_balance: 4)
+
+    _result = AddressSumMinusBurnt.get_sum_minus_burnt()
+
+    Process.sleep(1000)
+
+    updated_value = Decimal.to_integer(AddressSumMinusBurnt.get_sum_minus_burnt())
+
+    assert updated_value == 6
+  end
+
+  test "does not update cache if cache period did not pass" do
+    insert(:address, fetched_coin_balance: 1)
+    insert(:address, fetched_coin_balance: 2)
+    insert(:address, fetched_coin_balance: 3)
+    insert(:address, hash: "0x0000000000000000000000000000000000000000", fetched_coin_balance: 4)
+
+    _result = AddressSumMinusBurnt.get_sum_minus_burnt()
+
+    Process.sleep(1000)
+
+    updated_value = Decimal.to_integer(AddressSumMinusBurnt.get_sum_minus_burnt())
+
+    assert updated_value == 6
+
+    insert(:address, fetched_coin_balance: 4)
+    insert(:address, fetched_coin_balance: 5)
+
+    _updated_value = AddressSumMinusBurnt.get_sum_minus_burnt()
+
+    Process.sleep(1000)
+
+    updated_value = Decimal.to_integer(AddressSumMinusBurnt.get_sum_minus_burnt())
+
+    assert updated_value == 6
+  end
+end

apps/explorer/test/explorer/chain/cache/address_sum_test.exs

@@ -12,13 +12,14 @@ defmodule Explorer.Chain.Cache.AddressSumTest do
   test "returns default address sum" do
     result = AddressSum.get_sum()
 
-    assert is_nil(result)
+    assert result == Decimal.new(0)
   end
 
   test "updates cache if initial value is zero" do
     insert(:address, fetched_coin_balance: 1)
     insert(:address, fetched_coin_balance: 2)
     insert(:address, fetched_coin_balance: 3)
+    insert(:address, hash: "0x0000000000000000000000000000000000000000", fetched_coin_balance: 4)
 
     _result = AddressSum.get_sum()
 
@@ -26,7 +27,7 @@ defmodule Explorer.Chain.Cache.AddressSumTest do
 
     updated_value = Decimal.to_integer(AddressSum.get_sum())
 
-    assert updated_value == 6
+    assert updated_value == 10
   end
 
   test "does not update cache if cache period did not pass" do

apps/explorer/test/explorer/chain/cache/pending_transactions_test.exs

@@ -9,9 +9,9 @@ defmodule Explorer.Chain.Cache.PendingTransactionsTest do
 
       PendingTransactions.update([transaction])
 
-      transaction_hash = transaction.hash
+      assert [%{hash: pending_transaction_hash}] = PendingTransactions.all()
 
-      assert [%{hash: transaction_hash}] = PendingTransactions.all()
+      assert transaction.hash == pending_transaction_hash
     end
   end
 end

apps/explorer/test/explorer/chain/import/runner/blocks_test.exs

@@ -7,7 +7,7 @@ defmodule Explorer.Chain.Import.Runner.BlocksTest do
 
   alias Ecto.Multi
   alias Explorer.Chain.Import.Runner.{Blocks, Transactions}
-  alias Explorer.Chain.{Address, Block, InternalTransaction, Log, Transaction, TokenTransfer}
+  alias Explorer.Chain.{Address, Block, Transaction, TokenTransfer}
   alias Explorer.{Chain, Repo}
 
   describe "run/1" do
@@ -115,78 +115,6 @@ defmodule Explorer.Chain.Import.Runner.BlocksTest do
       assert count(Address.CurrentTokenBalance) == count
     end
 
-    test "remove_nonconsensus_token_transfers deletes token transfer rows with matching block number when new consensus block is inserted",
-         %{consensus_block: %{number: block_number} = block, options: options} do
-      consensus_block = insert(:block, number: block_number, consensus: true)
-
-      transaction = insert(:transaction) |> with_block(consensus_block)
-
-      %TokenTransfer{transaction_hash: transaction_hash, log_index: log_index} =
-        insert(:token_transfer, block_number: block_number, transaction: transaction)
-
-      assert count(TokenTransfer) == 1
-
-      assert {:ok,
-              %{
-                remove_nonconsensus_token_transfers: [
-                  %{transaction_hash: ^transaction_hash, log_index: ^log_index}
-                ]
-              }} = run_block_consensus_change(block, true, options)
-
-      assert count(TokenTransfer) == 0
-    end
-
-    test "remove_nonconsensus_token_transfers does not delete token transfer rows with matching block number when new consensus block wasn't inserted",
-         %{consensus_block: %{number: block_number} = block, options: options} do
-      consensus_block = insert(:block, number: block_number, consensus: true)
-
-      transaction = insert(:transaction) |> with_block(consensus_block)
-
-      insert(:token_transfer, block_number: block_number, transaction: transaction)
-
-      count = 1
-
-      assert count(TokenTransfer) == count
-
-      assert {:ok, %{remove_nonconsensus_token_transfers: []}} = run_block_consensus_change(block, false, options)
-
-      assert count(TokenTransfer) == count
-    end
-
-    test "remove_nonconsensus_logs deletes nonconsensus logs", %{
-      consensus_block: %{number: block_number} = block,
-      options: options
-    } do
-      old_block = insert(:block, number: block_number, consensus: true)
-      forked_transaction = :transaction |> insert() |> with_block(old_block)
-      %Log{transaction_hash: hash, index: index} = insert(:log, transaction: forked_transaction)
-
-      assert count(Log) == 1
-
-      assert {:ok, %{remove_nonconsensus_logs: [%{transaction_hash: ^hash, index: ^index}]}} =
-               run_block_consensus_change(block, true, options)
-
-      assert count(Log) == 0
-    end
-
-    test "remove_nonconsensus_internal_transactions deletes nonconsensus internal transactions", %{
-      consensus_block: %{number: block_number} = block,
-      options: options
-    } do
-      old_block = insert(:block, number: block_number, consensus: true)
-      forked_transaction = :transaction |> insert() |> with_block(old_block)
-
-      %InternalTransaction{index: index, transaction_hash: hash} =
-        insert(:internal_transaction, index: 0, transaction: forked_transaction)
-
-      assert count(InternalTransaction) == 1
-
-      assert {:ok, %{remove_nonconsensus_internal_transactions: [%{transaction_hash: ^hash, index: ^index}]}} =
-               run_block_consensus_change(block, true, options)
-
-      assert count(InternalTransaction) == 0
-    end
-
     test "derive_address_current_token_balances inserts rows if there is an address_token_balance left for the rows deleted by delete_address_current_token_balances",
          %{consensus_block: %{number: block_number} = block, options: options} do
       token = insert(:token)

apps/explorer/test/explorer/chain/import/runner/internal_transactions_test.exs

@@ -2,19 +2,20 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactionsTest do
   use Explorer.DataCase
 
   alias Ecto.Multi
-  alias Explorer.Chain.{Block, Data, Wei, Transaction, InternalTransaction}
+  alias Explorer.Chain.{Block, Data, Wei, PendingBlockOperation, Transaction, InternalTransaction}
   alias Explorer.Chain.Import.Runner.InternalTransactions
 
   describe "run/1" do
     test "transaction's status becomes :error when its internal_transaction has an error" do
       transaction = insert(:transaction) |> with_block(status: :ok)
+      insert(:pending_block_operation, block_hash: transaction.block_hash, fetch_internal_transactions: true)
 
       assert :ok == transaction.status
 
       index = 0
       error = "Reverted"
 
-      internal_transaction_changes = make_internal_transaction_changes(transaction.hash, index, error)
+      internal_transaction_changes = make_internal_transaction_changes(transaction, index, error)
 
       assert {:ok, _} = run_internal_transactions([internal_transaction_changes])
 
@@ -25,18 +26,21 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactionsTest do
       transaction = insert(:transaction) |> with_block(status: :ok)
       pending = insert(:transaction)
 
+      insert(:pending_block_operation, block_hash: transaction.block_hash, fetch_internal_transactions: true)
+
       assert :ok == transaction.status
       assert is_nil(pending.block_hash)
 
       index = 0
 
-      transaction_changes = make_internal_transaction_changes(transaction.hash, index, nil)
+      transaction_changes = make_internal_transaction_changes(transaction, index, nil)
-      pending_changes = make_internal_transaction_changes(pending.hash, index, nil)
+      pending_changes = make_internal_transaction_changes(pending, index, nil)
 
       assert {:ok, _} = run_internal_transactions([transaction_changes, pending_changes])
 
-      assert %InternalTransaction{} =
+      assert Repo.exists?(from(i in InternalTransaction, where: i.transaction_hash == ^transaction.hash))
-               Repo.one(from(i in InternalTransaction, where: i.transaction_hash == ^transaction.hash))
+
+      assert PendingBlockOperation |> Repo.get(transaction.block_hash) |> is_nil()
 
       assert from(i in InternalTransaction, where: i.transaction_hash == ^pending.hash) |> Repo.one() |> is_nil()
 
@@ -47,68 +51,112 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactionsTest do
       empty_block = insert(:block)
       pending = insert(:transaction)
 
+      insert(:pending_block_operation, block_hash: empty_block.hash, fetch_internal_transactions: true)
+
       assert is_nil(pending.block_hash)
 
       full_block = insert(:block)
       inserted = insert(:transaction) |> with_block(full_block)
 
+      insert(:pending_block_operation, block_hash: full_block.hash, fetch_internal_transactions: true)
+
       assert full_block.hash == inserted.block_hash
 
       index = 0
 
       pending_transaction_changes =
-        pending.hash
+        pending
         |> make_internal_transaction_changes(index, nil)
         |> Map.put(:block_number, empty_block.number)
 
-      transaction_changes =
+      transaction_changes = make_internal_transaction_changes(inserted, index, nil)
-        inserted.hash
-        |> make_internal_transaction_changes(index, nil)
-        |> Map.put(:block_number, full_block.number)
-
-      multi =
-        Multi.new()
-        |> Multi.run(:internal_transactions_indexed_at_blocks, fn _, _ -> {:ok, [empty_block.hash, full_block.hash]} end)
 
-      assert {:ok, _} = run_internal_transactions([pending_transaction_changes, transaction_changes], multi)
+      assert {:ok, _} = run_internal_transactions([pending_transaction_changes, transaction_changes])
 
       assert from(i in InternalTransaction, where: i.transaction_hash == ^pending.hash) |> Repo.one() |> is_nil()
 
       assert %{consensus: false} = Repo.get(Block, empty_block.hash)
+      assert not is_nil(Repo.get(PendingBlockOperation, empty_block.hash))
 
       assert from(i in InternalTransaction, where: i.transaction_hash == ^inserted.hash) |> Repo.one() |> is_nil() ==
                false
 
       assert %{consensus: true} = Repo.get(Block, full_block.hash)
+      assert PendingBlockOperation |> Repo.get(full_block.hash) |> is_nil()
+    end
+
+    test "removes old records with the same primary key (transaction_hash, index)" do
+      full_block = insert(:block)
+      another_full_block = insert(:block)
+
+      transaction = insert(:transaction) |> with_block(full_block)
+
+      insert(:internal_transaction,
+        index: 0,
+        transaction: transaction,
+        block_hash: another_full_block.hash,
+        block_index: 0
+      )
+
+      insert(:pending_block_operation, block_hash: full_block.hash, fetch_internal_transactions: true)
+
+      transaction_changes = make_internal_transaction_changes(transaction, 0, nil)
+
+      assert {:ok, %{remove_left_over_internal_transactions: {1, nil}}} =
+               run_internal_transactions([transaction_changes])
+
+      assert from(i in InternalTransaction,
+               where: i.transaction_hash == ^transaction.hash and i.block_hash == ^another_full_block.hash
+             )
+             |> Repo.one()
+             |> is_nil()
+    end
+
+    test "removes consensus to blocks where not all transactions are filled" do
+      full_block = insert(:block)
+      transaction_a = insert(:transaction) |> with_block(full_block)
+      transaction_b = insert(:transaction) |> with_block(full_block)
+
+      insert(:pending_block_operation, block_hash: full_block.hash, fetch_internal_transactions: true)
+
+      transaction_a_changes = make_internal_transaction_changes(transaction_a, 0, nil)
+
+      assert {:ok, _} = run_internal_transactions([transaction_a_changes])
+
+      assert from(i in InternalTransaction, where: i.transaction_hash == ^transaction_a.hash) |> Repo.one() |> is_nil()
+      assert from(i in InternalTransaction, where: i.transaction_hash == ^transaction_b.hash) |> Repo.one() |> is_nil()
+
+      assert %{consensus: false} = Repo.get(Block, full_block.hash)
+      assert not is_nil(Repo.get(PendingBlockOperation, full_block.hash))
     end
 
     test "does not remove consensus when block is empty and no transactions are missing" do
       empty_block = insert(:block)
 
+      insert(:pending_block_operation, block_hash: empty_block.hash, fetch_internal_transactions: true)
+
       full_block = insert(:block)
       inserted = insert(:transaction) |> with_block(full_block)
 
+      insert(:pending_block_operation, block_hash: full_block.hash, fetch_internal_transactions: true)
+
       assert full_block.hash == inserted.block_hash
 
       index = 0
 
-      transaction_changes =
+      transaction_changes = make_internal_transaction_changes(inserted, index, nil)
-        inserted.hash
+      empty_changes = make_empty_block_changes(empty_block.number)
-        |> make_internal_transaction_changes(index, nil)
-        |> Map.put(:block_number, full_block.number)
-
-      multi =
-        Multi.new()
-        |> Multi.run(:internal_transactions_indexed_at_blocks, fn _, _ -> {:ok, [empty_block.hash, full_block.hash]} end)
 
-      assert {:ok, _} = run_internal_transactions([transaction_changes], multi)
+      assert {:ok, _} = run_internal_transactions([empty_changes, transaction_changes])
 
       assert %{consensus: true} = Repo.get(Block, empty_block.hash)
+      assert PendingBlockOperation |> Repo.get(empty_block.hash) |> is_nil()
 
       assert from(i in InternalTransaction, where: i.transaction_hash == ^inserted.hash) |> Repo.one() |> is_nil() ==
                false
 
       assert %{consensus: true} = Repo.get(Block, full_block.hash)
+      assert PendingBlockOperation |> Repo.get(full_block.hash) |> is_nil()
     end
   end
 
@@ -121,7 +169,9 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactionsTest do
     |> Repo.transaction()
   end
 
-  defp make_internal_transaction_changes(transaction_hash, index, error) do
+  defp make_empty_block_changes(block_number), do: %{block_number: block_number}
+
+  defp make_internal_transaction_changes(transaction, index, error) do
     %{
       from_address_hash: insert(:address).hash,
       to_address_hash: insert(:address).hash,
@@ -142,10 +192,11 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactionsTest do
         end,
       index: index,
       trace_address: [],
-      transaction_hash: transaction_hash,
+      transaction_hash: transaction.hash,
       type: :call,
       value: Wei.from(Decimal.new(1), :wei),
-      error: error
+      error: error,
+      block_number: transaction.block_number
     }
   end
 end

apps/explorer/test/explorer/chain/import_test.exs

@@ -12,6 +12,7 @@ defmodule Explorer.Chain.ImportTest do
     Log,
     Hash,
     Import,
+    PendingBlockOperation,
     Token,
     TokenTransfer,
     Transaction
@@ -81,11 +82,13 @@ defmodule Explorer.Chain.ImportTest do
             value: 0
           }
         ],
-        timeout: 5
+        timeout: 5,
+        with: :blockless_changeset
       },
       logs: %{
         params: [
           %{
+            block_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd",
             address_hash: "0x8bf38d4764929064f2d4d3a56520a76ab3df415b",
             data: "0x0000000000000000000000000000000000000000000000000de0b6b3a7640000",
             first_topic: "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef",
@@ -149,6 +152,7 @@ defmodule Explorer.Chain.ImportTest do
           %{
             amount: Decimal.new(1_000_000_000_000_000_000),
             block_number: 37,
+            block_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd",
             log_index: 0,
             from_address_hash: "0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca",
             to_address_hash: "0x515c09c5bba1ed566b02a5b0599ec5d5d0aee73d",
@@ -305,9 +309,7 @@ defmodule Explorer.Chain.ImportTest do
                       bytes:
                         <<83, 189, 136, 72, 114, 222, 62, 72, 134, 146, 136, 27, 174, 236, 38, 46, 123, 149, 35, 77, 57,
                           101, 36, 140, 57, 254, 153, 47, 255, 212, 51, 229>>
-                    },
+                    }
-                    # because there are successful, non-contract-creation token transfer
-                    internal_transactions_indexed_at: %DateTime{}
                   }
                 ],
                 tokens: [
@@ -554,7 +556,8 @@ defmodule Explorer.Chain.ImportTest do
               block_number: 37,
               transaction_index: 0
             }
-          ]
+          ],
+          with: :blockless_changeset
         }
       }
 
@@ -565,63 +568,7 @@ defmodule Explorer.Chain.ImportTest do
       assert address.contract_code != smart_contract_bytecode
     end
 
-    test "updates `error`, `status` and `internal_transaction_indexed_at` even if internal transactions were alreader inserted" do
+    test "with internal_transactions updates PendingBlockOperation status" do
-      address_hash = "0x1c494fa496f1cfd918b5ff190835af3aaf609899"
-      from_address = insert(:address, hash: address_hash)
-
-      block = insert(:block, consensus: true, number: 37)
-
-      transaction =
-        :transaction
-        |> insert(error: nil, internal_transactions_indexed_at: nil, status: nil, from_address: from_address)
-        |> with_block(block, status: :error)
-
-      internal_transacton =
-        insert(:internal_transaction,
-          block_number: 37,
-          transaction_hash: transaction.hash,
-          error: "Bad Instruction",
-          index: 0,
-          gas_used: nil,
-          output: nil,
-          gas: 19,
-          type: "call"
-        )
-
-      options = %{
-        internal_transactions: %{
-          params: [
-            %{
-              block_number: internal_transacton.block_number,
-              call_type: internal_transacton.type,
-              gas: internal_transacton.gas,
-              gas_used: internal_transacton.gas_used,
-              index: internal_transacton.index,
-              output: internal_transacton.output,
-              transaction_hash: internal_transacton.transaction_hash,
-              type: internal_transacton.type,
-              from_address_hash: address_hash,
-              to_address_hash: address_hash,
-              trace_address: [],
-              value: 0,
-              transaction_index: 0,
-              error: internal_transacton.error,
-              input: internal_transacton.input
-            }
-          ]
-        }
-      }
-
-      {:ok, _} = Import.all(options)
-
-      assert result =
-               %Transaction{error: "Bad Instruction", status: :error} =
-               Repo.one!(from(t in Transaction, where: t.hash == ^transaction.hash))
-
-      assert result.internal_transactions_indexed_at
-    end
-
-    test "with internal_transactions updates Transaction internal_transactions_indexed_at" do
       block_hash = "0xe52d77084cab13a4e724162bcd8c6028e5ecfaa04d091ee476e96b9958ed6b47"
       block_number = 34
       miner_hash = from_address_hash = "0xe8ddc5c7a2d2f0d7a9798459c0104fdf5e987aca"
@@ -674,7 +621,10 @@ defmodule Explorer.Chain.ImportTest do
               value: 0
             }
           ]
-        },
+        }
+      }
+
+      internal_txs_options = %{
         internal_transactions: %{
           params: [
             %{
@@ -693,17 +643,18 @@ defmodule Explorer.Chain.ImportTest do
               output: "0x",
               value: 0
             }
-          ]
+          ],
+          with: :blockless_changeset
         }
       }
 
-      refute Enum.any?(options[:transactions][:params], &Map.has_key?(&1, :internal_transactions_indexed_at))
-
       assert {:ok, _} = Import.all(options)
 
-      transaction = Explorer.Repo.get(Transaction, transaction_hash)
+      assert [block_hash] = Explorer.Repo.all(PendingBlockOperation.block_hashes(:fetch_internal_transactions))
 
-      refute transaction.internal_transactions_indexed_at == nil
+      assert {:ok, _} = Import.all(internal_txs_options)
+
+      assert [] == Explorer.Repo.all(PendingBlockOperation.block_hashes(:fetch_internal_transactions))
     end
 
     test "when the transaction has no to_address and an internal transaction with type create it populates the denormalized created_contract_address_hash" do
@@ -786,7 +737,8 @@ defmodule Explorer.Chain.ImportTest do
               value: 0,
               transaction_index: 0
             }
-          ]
+          ],
+          with: :blockless_changeset
         }
       }
 
@@ -896,7 +848,8 @@ defmodule Explorer.Chain.ImportTest do
                        transaction_index: 1
                      }
                    ],
-                   timeout: 5
+                   timeout: 5,
+                   with: :blockless_changeset
                  },
                  addresses: %{
                    params: [
@@ -1080,7 +1033,8 @@ defmodule Explorer.Chain.ImportTest do
                        transaction_index: 0,
                        transaction_block_number: 35
                      }
-                   ]
+                   ],
+                   with: :blockless_changeset
                  }
                })
 
@@ -1561,16 +1515,24 @@ defmodule Explorer.Chain.ImportTest do
                        transaction_index: 0
                      )
                    ],
-                   timeout: 1
+                   timeout: 1,
+                   with: :blockless_changeset
                  },
                  logs: %{
-                   params: [params_for(:log, transaction_hash: transaction_hash, address_hash: miner_hash)],
+                   params: [
+                     params_for(:log,
+                       transaction_hash: transaction_hash,
+                       address_hash: miner_hash,
+                       block_hash: block_hash
+                     )
+                   ],
                    timeout: 1
                  },
                  token_transfers: %{
                    params: [
                      params_for(
                        :token_transfer,
+                       block_hash: block_hash,
                        block_number: 35,
                        from_address_hash: from_address_hash,
                        to_address_hash: to_address_hash,
@@ -1792,7 +1754,6 @@ defmodule Explorer.Chain.ImportTest do
                        block_hash: block_hash_before,
                        block_number: block_number,
                        error: error,
-                       internal_transactions_indexed_at: Timex.parse!("2019-01-01T01:00:00Z", "{ISO:Extended:Z}"),
                        from_address_hash: from_address_hash_before,
                        to_address_hash: to_address_hash_before,
                        gas: 21_000,
@@ -1828,7 +1789,8 @@ defmodule Explorer.Chain.ImportTest do
                        block_number: block_number,
                        transaction_index: 0
                      }
-                   ]
+                   ],
+                   with: :blockless_changeset
                  }
                })
 

apps/explorer/test/explorer/chain/internal_transaction_test.exs

@@ -26,7 +26,9 @@ defmodule Explorer.Chain.InternalTransactionTest do
           transaction_hash: transaction.hash,
           type: "call",
           value: 100,
-          block_number: 35
+          block_number: 35,
+          block_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd",
+          block_index: 0
         })
 
       assert changeset.valid?

apps/explorer/test/explorer/chain/log_test.exs

@@ -9,7 +9,12 @@ defmodule Explorer.Chain.LogTest do
 
   describe "changeset/2" do
     test "accepts valid attributes" do
-      params = params_for(:log, address_hash: build(:address).hash, transaction_hash: build(:transaction).hash)
+      params =
+        params_for(:log,
+          address_hash: build(:address).hash,
+          transaction_hash: build(:transaction).hash,
+          block_hash: build(:block).hash
+        )
 
       assert %Changeset{valid?: true} = Log.changeset(%Log{}, params)
     end
@@ -26,7 +31,8 @@ defmodule Explorer.Chain.LogTest do
           :log,
           address_hash: build(:address).hash,
           first_topic: "ham",
-          transaction_hash: build(:transaction).hash
+          transaction_hash: build(:transaction).hash,
+          block_hash: build(:block).hash
         )
 
       assert %Changeset{changes: %{first_topic: "ham"}, valid?: true} = Log.changeset(%Log{}, params)

apps/explorer/test/explorer/chain_test.exs

@@ -18,6 +18,7 @@ defmodule Explorer.ChainTest do
     Hash,
     InternalTransaction,
     Log,
+    PendingBlockOperation,
     Token,
     TokenTransfer,
     Transaction,
@@ -35,6 +36,38 @@ defmodule Explorer.ChainTest do
 
   setup :verify_on_exit!
 
+  describe "remove_nonconsensus_blocks_from_pending_ops/0" do
+    test "removes pending ops for nonconsensus blocks" do
+      block = insert(:block)
+      insert(:pending_block_operation, block: block, fetch_internal_transactions: true)
+
+      nonconsensus_block = insert(:block, consensus: false)
+      insert(:pending_block_operation, block: nonconsensus_block, fetch_internal_transactions: true)
+
+      :ok = Chain.remove_nonconsensus_blocks_from_pending_ops()
+
+      assert Repo.get(PendingBlockOperation, block.hash)
+      assert is_nil(Repo.get(PendingBlockOperation, nonconsensus_block.hash))
+    end
+
+    test "removes pending ops for nonconsensus blocks by block hashes" do
+      block = insert(:block)
+      insert(:pending_block_operation, block: block, fetch_internal_transactions: true)
+
+      nonconsensus_block = insert(:block, consensus: false)
+      insert(:pending_block_operation, block: nonconsensus_block, fetch_internal_transactions: true)
+
+      nonconsensus_block1 = insert(:block, consensus: false)
+      insert(:pending_block_operation, block: nonconsensus_block1, fetch_internal_transactions: true)
+
+      :ok = Chain.remove_nonconsensus_blocks_from_pending_ops([nonconsensus_block1.hash])
+
+      assert Repo.get(PendingBlockOperation, block.hash)
+      assert Repo.get(PendingBlockOperation, nonconsensus_block.hash)
+      assert is_nil(Repo.get(PendingBlockOperation, nonconsensus_block1.hash))
+    end
+  end
+
   describe "count_addresses_with_balance_from_cache/0" do
     test "returns the number of addresses with fetched_coin_balance > 0" do
       insert(:address, fetched_coin_balance: 0)
@@ -220,14 +253,26 @@ defmodule Explorer.ChainTest do
         |> insert(to_address: address)
         |> with_block()
 
-      insert(:log, transaction: transaction1, index: 1, address: address)
+      insert(:log,
+        block: transaction1.block,
+        block_number: transaction1.block_number,
+        transaction: transaction1,
+        index: 1,
+        address: address
+      )
 
       transaction2 =
         :transaction
         |> insert(from_address: address)
         |> with_block()
 
-      insert(:log, transaction: transaction2, index: 2, address: address)
+      insert(:log,
+        block: transaction2.block,
+        block_number: transaction2.block_number,
+        transaction: transaction2,
+        index: 2,
+        address: address
+      )
 
       assert Enum.count(Chain.address_to_logs(address_hash)) == 2
     end
@@ -240,10 +285,18 @@ defmodule Explorer.ChainTest do
         |> insert(to_address: address)
         |> with_block()
 
-      log1 = insert(:log, transaction: transaction, index: 1, address: address)
+      log1 = insert(:log, transaction: transaction, index: 1, address: address, block_number: transaction.block_number)
 
       2..51
-      |> Enum.map(fn index -> insert(:log, transaction: transaction, index: index, address: address) end)
+      |> Enum.map(fn index ->
+        insert(:log,
+          block: transaction.block,
+          transaction: transaction,
+          index: index,
+          address: address,
+          block_number: transaction.block_number
+        )
+      end)
       |> Enum.map(& &1.index)
 
       paging_options1 = %PagingOptions{page_size: 1}
@@ -263,14 +316,27 @@ defmodule Explorer.ChainTest do
         |> insert(to_address: address)
         |> with_block()
 
-      insert(:log, transaction: transaction1, index: 1, address: address)
+      insert(:log,
+        block: transaction1.block,
+        transaction: transaction1,
+        index: 1,
+        address: address,
+        block_number: transaction1.block_number
+      )
 
       transaction2 =
         :transaction
         |> insert(from_address: address)
         |> with_block()
 
-      insert(:log, transaction: transaction2, index: 2, address: address, first_topic: "test")
+      insert(:log,
+        block: transaction2.block,
+        transaction: transaction2,
+        index: 2,
+        address: address,
+        first_topic: "test",
+        block_number: transaction2.block_number
+      )
 
       [found_log] = Chain.address_to_logs(address_hash, topic: "test")
 
@@ -285,14 +351,27 @@ defmodule Explorer.ChainTest do
         |> insert(to_address: address)
         |> with_block()
 
-      insert(:log, transaction: transaction1, index: 1, address: address, fourth_topic: "test")
+      insert(:log,
+        block: transaction1.block,
+        block_number: transaction1.block_number,
+        transaction: transaction1,
+        index: 1,
+        address: address,
+        fourth_topic: "test"
+      )
 
       transaction2 =
         :transaction
         |> insert(from_address: address)
         |> with_block()
 
-      insert(:log, transaction: transaction2, index: 2, address: address)
+      insert(:log,
+        block: transaction2.block,
+        block_number: transaction2.block.number,
+        transaction: transaction2,
+        index: 2,
+        address: address
+      )
 
       [found_log] = Chain.address_to_logs(address_hash, topic: "test")
 
@@ -393,6 +472,8 @@ defmodule Explorer.ChainTest do
           transaction: transaction,
           index: 0,
           block_number: transaction.block_number,
+          block_hash: transaction.block_hash,
+          block_index: 0,
           transaction_index: transaction.index
         )
 
@@ -835,7 +916,7 @@ defmodule Explorer.ChainTest do
 
       :transaction
       |> insert()
-      |> with_block(block, internal_transactions_indexed_at: DateTime.utc_now())
+      |> with_block(block)
 
       assert Chain.finished_indexing?()
     end
@@ -851,6 +932,8 @@ defmodule Explorer.ChainTest do
       |> insert()
       |> with_block(block)
 
+      insert(:pending_block_operation, block: block, fetch_internal_transactions: true)
+
       refute Chain.finished_indexing?()
     end
   end
@@ -948,6 +1031,8 @@ defmodule Explorer.ChainTest do
         transaction: transaction,
         index: 0,
         block_number: transaction.block_number,
+        block_hash: transaction.block_hash,
+        block_index: 0,
         transaction_index: transaction.index
       )
 
@@ -956,6 +1041,8 @@ defmodule Explorer.ChainTest do
           transaction: transaction,
           index: index,
           block_number: transaction.block_number,
+          block_hash: transaction.block_hash,
+          block_index: index,
           transaction_index: transaction.index
         )
       end)
@@ -1310,11 +1397,13 @@ defmodule Explorer.ChainTest do
             output: "0x",
             value: 0
           }
-        ]
+        ],
+        with: :blockless_changeset
       },
       logs: %{
         params: [
           %{
+            block_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd",
             address_hash: "0x8bf38d4764929064f2d4d3a56520a76ab3df415b",
             data: "0x0000000000000000000000000000000000000000000000000de0b6b3a7640000",
             first_topic: "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef",
@@ -1372,6 +1461,7 @@ defmodule Explorer.ChainTest do
       token_transfers: %{
         params: [
           %{
+            block_hash: "0xf6b4b8c88df3ebd252ec476328334dc026cf66606a84fb769b3d3cbccc8471bd",
             amount: Decimal.new(1_000_000_000_000_000_000),
             block_number: 37,
             log_index: 0,
@@ -1521,9 +1611,7 @@ defmodule Explorer.ChainTest do
                       bytes:
                         <<83, 189, 136, 72, 114, 222, 62, 72, 134, 146, 136, 27, 174, 236, 38, 46, 123, 149, 35, 77, 57,
                           101, 36, 140, 57, 254, 153, 47, 255, 212, 51, 229>>
-                    },
+                    }
-                    # because there are successful, non-contract-creation token transfer
-                    internal_transactions_indexed_at: %DateTime{}
                   }
                 ],
                 tokens: [
@@ -1797,6 +1885,8 @@ defmodule Explorer.ChainTest do
           transaction: transaction,
           to_address: address,
           block_number: transaction.block_number,
+          block_hash: transaction.block_hash,
+          block_index: 1,
           transaction_index: transaction.index
         )
 
@@ -1806,6 +1896,8 @@ defmodule Explorer.ChainTest do
           transaction: transaction,
           to_address: address,
           block_number: transaction.block_number,
+          block_hash: transaction.block_hash,
+          block_index: 2,
           transaction_index: transaction.index
         )
 
@@ -1832,6 +1924,8 @@ defmodule Explorer.ChainTest do
         to_address: address,
         index: 0,
         block_number: transaction.block_number,
+        block_hash: transaction.block_hash,
+        block_index: 0,
         transaction_index: transaction.index
       )
 
@@ -1840,6 +1934,8 @@ defmodule Explorer.ChainTest do
         to_address: address,
         index: 1,
         block_number: transaction.block_number,
+        block_hash: transaction.block_hash,
+        block_index: 1,
         transaction_index: transaction.index
       )
 
@@ -1887,6 +1983,8 @@ defmodule Explorer.ChainTest do
           to_address: address,
           index: 1,
           block_number: pending_transaction.block_number,
+          block_hash: pending_transaction.block_hash,
+          block_index: 1,
           transaction_index: pending_transaction.index
         )
 
@@ -1897,6 +1995,8 @@ defmodule Explorer.ChainTest do
           to_address: address,
           index: 2,
           block_number: pending_transaction.block_number,
+          block_hash: pending_transaction.block_hash,
+          block_index: 2,
           transaction_index: pending_transaction.index
         )
 
@@ -1914,6 +2014,8 @@ defmodule Explorer.ChainTest do
           to_address: address,
           index: 1,
           block_number: first_a_transaction.block_number,
+          block_hash: a_block.hash,
+          block_index: 1,
           transaction_index: first_a_transaction.index
         )
 
@@ -1924,6 +2026,8 @@ defmodule Explorer.ChainTest do
           to_address: address,
           index: 2,
           block_number: first_a_transaction.block_number,
+          block_hash: a_block.hash,
+          block_index: 2,
           transaction_index: first_a_transaction.index
         )
 
@@ -1939,6 +2043,8 @@ defmodule Explorer.ChainTest do
           to_address: address,
           index: 1,
           block_number: second_a_transaction.block_number,
+          block_hash: a_block.hash,
+          block_index: 4,
           transaction_index: second_a_transaction.index
         )
 
@@ -1949,6 +2055,8 @@ defmodule Explorer.ChainTest do
           to_address: address,
           index: 2,
           block_number: second_a_transaction.block_number,
+          block_hash: a_block.hash,
+          block_index: 5,
           transaction_index: second_a_transaction.index
         )
 
@@ -1966,6 +2074,8 @@ defmodule Explorer.ChainTest do
           to_address: address,
           index: 1,
           block_number: first_b_transaction.block_number,
+          block_hash: b_block.hash,
+          block_index: 1,
           transaction_index: first_b_transaction.index
         )
 
@@ -1976,6 +2086,8 @@ defmodule Explorer.ChainTest do
           to_address: address,
           index: 2,
           block_number: first_b_transaction.block_number,
+          block_hash: b_block.hash,
+          block_index: 2,
           transaction_index: first_b_transaction.index
         )
 
@@ -2001,10 +2113,14 @@ defmodule Explorer.ChainTest do
 
       pending_transaction = insert(:transaction)
 
+      old_block = insert(:block, consensus: false)
+
       insert(
         :internal_transaction,
         transaction: pending_transaction,
         to_address: address,
+        block_hash: old_block.hash,
+        block_index: 1,
         index: 1
       )
 
@@ -2012,6 +2128,8 @@ defmodule Explorer.ChainTest do
         :internal_transaction,
         transaction: pending_transaction,
         to_address: address,
+        block_hash: old_block.hash,
+        block_index: 2,
         index: 2
       )
 
@@ -2029,6 +2147,8 @@ defmodule Explorer.ChainTest do
           to_address: address,
           index: 1,
           block_number: first_a_transaction.block_number,
+          block_hash: a_block.hash,
+          block_index: 1,
           transaction_index: first_a_transaction.index
         )
 
@@ -2039,6 +2159,8 @@ defmodule Explorer.ChainTest do
           to_address: address,
           index: 2,
           block_number: first_a_transaction.block_number,
+          block_hash: a_block.hash,
+          block_index: 2,
           transaction_index: first_a_transaction.index
         )
 
@@ -2054,6 +2176,8 @@ defmodule Explorer.ChainTest do
           to_address: address,
           index: 1,
           block_number: second_a_transaction.block_number,
+          block_hash: a_block.hash,
+          block_index: 4,
           transaction_index: second_a_transaction.index
         )
 
@@ -2064,6 +2188,8 @@ defmodule Explorer.ChainTest do
           to_address: address,
           index: 2,
           block_number: second_a_transaction.block_number,
+          block_hash: a_block.hash,
+          block_index: 5,
           transaction_index: second_a_transaction.index
         )
 
@@ -2081,6 +2207,8 @@ defmodule Explorer.ChainTest do
           to_address: address,
           index: 1,
           block_number: first_b_transaction.block_number,
+          block_hash: b_block.hash,
+          block_index: 1,
           transaction_index: first_b_transaction.index
         )
 
@@ -2091,6 +2219,8 @@ defmodule Explorer.ChainTest do
           to_address: address,
           index: 2,
           block_number: first_b_transaction.block_number,
+          block_hash: b_block.hash,
+          block_index: 2,
           transaction_index: first_b_transaction.index
         )
 
@@ -2158,6 +2288,8 @@ defmodule Explorer.ChainTest do
         to_address: address,
         transaction: transaction,
         block_number: transaction.block_number,
+        block_hash: transaction.block_hash,
+        block_index: 0,
         transaction_index: transaction.index
       )
 
@@ -2178,6 +2310,8 @@ defmodule Explorer.ChainTest do
           index: 0,
           from_address: address,
           transaction: transaction,
+          block_hash: transaction.block_hash,
+          block_index: 0,
           block_number: transaction.block_number,
           transaction_index: transaction.index
         )
@@ -2241,6 +2375,8 @@ defmodule Explorer.ChainTest do
           transaction: transaction,
           index: 0,
           block_number: transaction.block_number,
+          block_hash: transaction.block_hash,
+          block_index: 0,
           transaction_index: transaction.index
         )
 
@@ -2248,6 +2384,8 @@ defmodule Explorer.ChainTest do
         insert(:internal_transaction,
           transaction: transaction,
           index: 1,
+          block_hash: transaction.block_hash,
+          block_index: 1,
           block_number: transaction.block_number,
           transaction_index: transaction.index
         )
@@ -2277,6 +2415,8 @@ defmodule Explorer.ChainTest do
         transaction: transaction,
         index: 0,
         block_number: transaction.block_number,
+        block_hash: transaction.block_hash,
+        block_index: 0,
         transaction_index: transaction.index
       )
 
@@ -2315,6 +2455,8 @@ defmodule Explorer.ChainTest do
         transaction: transaction,
         index: 0,
         block_number: transaction.block_number,
+        block_hash: transaction.block_hash,
+        block_index: 0,
         transaction_index: transaction.index
       )
 
@@ -2334,6 +2476,8 @@ defmodule Explorer.ChainTest do
           index: 0,
           transaction: transaction,
           block_number: transaction.block_number,
+          block_hash: transaction.block_hash,
+          block_index: 0,
           transaction_index: transaction.index
         )
 
@@ -2354,6 +2498,8 @@ defmodule Explorer.ChainTest do
           transaction: transaction,
           type: :reward,
           block_number: transaction.block_number,
+          block_hash: transaction.block_hash,
+          block_index: 0,
           transaction_index: transaction.index
         )
 
@@ -2375,6 +2521,8 @@ defmodule Explorer.ChainTest do
           gas: nil,
           type: :selfdestruct,
           block_number: transaction.block_number,
+          block_hash: transaction.block_hash,
+          block_index: 0,
           transaction_index: transaction.index
         )
 
@@ -2394,6 +2542,8 @@ defmodule Explorer.ChainTest do
           transaction: transaction,
           index: 0,
           block_number: transaction.block_number,
+          block_hash: transaction.block_hash,
+          block_index: 0,
           transaction_index: transaction.index
         )
 
@@ -2402,6 +2552,8 @@ defmodule Explorer.ChainTest do
           transaction: transaction,
           index: 1,
           block_number: transaction.block_number,
+          block_hash: transaction.block_hash,
+          block_index: 1,
           transaction_index: transaction.index
         )
 
@@ -2424,6 +2576,8 @@ defmodule Explorer.ChainTest do
           transaction: transaction,
           index: 0,
           block_number: transaction.block_number,
+          block_hash: transaction.block_hash,
+          block_index: 0,
           transaction_index: transaction.index
         )
 
@@ -2432,6 +2586,8 @@ defmodule Explorer.ChainTest do
           transaction: transaction,
           index: 1,
           block_number: transaction.block_number,
+          block_hash: transaction.block_hash,
+          block_index: 1,
           transaction_index: transaction.index
         )
 
@@ -2465,7 +2621,8 @@ defmodule Explorer.ChainTest do
         |> insert()
         |> with_block()
 
-      %Log{transaction_hash: transaction_hash, index: index} = insert(:log, transaction: transaction)
+      %Log{transaction_hash: transaction_hash, index: index} =
+        insert(:log, transaction: transaction, block: transaction.block, block_number: transaction.block_number)
 
       assert [%Log{transaction_hash: ^transaction_hash, index: ^index}] = Chain.transaction_to_logs(transaction.hash)
     end
@@ -2476,11 +2633,24 @@ defmodule Explorer.ChainTest do
         |> insert()
         |> with_block()
 
-      log = insert(:log, transaction: transaction, index: 1)
+      log =
+        insert(:log,
+          transaction: transaction,
+          index: 1,
+          block: transaction.block,
+          block_number: transaction.block_number
+        )
 
       second_page_indexes =
         2..51
-        |> Enum.map(fn index -> insert(:log, transaction: transaction, index: index) end)
+        |> Enum.map(fn index ->
+          insert(:log,
+            transaction: transaction,
+            index: index,
+            block: transaction.block,
+            block_number: transaction.block_number
+          )
+        end)
         |> Enum.map(& &1.index)
 
       assert second_page_indexes ==
@@ -2495,7 +2665,7 @@ defmodule Explorer.ChainTest do
         |> insert()
         |> with_block()
 
-      insert(:log, transaction: transaction)
+      insert(:log, transaction: transaction, block: transaction.block, block_number: transaction.block_number)
 
       assert [%Log{address: %Address{}, transaction: %Transaction{}}] =
                Chain.transaction_to_logs(
@@ -2529,7 +2699,11 @@ defmodule Explorer.ChainTest do
         |> with_block()
 
       %TokenTransfer{transaction_hash: transaction_hash, log_index: log_index} =
-        insert(:token_transfer, transaction: transaction)
+        insert(:token_transfer,
+          transaction: transaction,
+          block: transaction.block,
+          block_number: transaction.block_number
+        )
 
       assert [%TokenTransfer{transaction_hash: ^transaction_hash, log_index: ^log_index}] =
                Chain.transaction_to_token_transfers(transaction.hash)
@@ -2541,7 +2715,7 @@ defmodule Explorer.ChainTest do
         |> insert()
         |> with_block()
 
-      insert(:token_transfer, transaction: transaction)
+      insert(:token_transfer, transaction: transaction, block: transaction.block, block_number: transaction.block_number)
 
       assert [%TokenTransfer{token: %Token{}, transaction: %Transaction{}}] =
                Chain.transaction_to_token_transfers(
@@ -2902,6 +3076,8 @@ defmodule Explorer.ChainTest do
         created_contract_address: created_contract_address,
         created_contract_code: smart_contract_bytecode,
         block_number: transaction.block_number,
+        block_hash: transaction.block_hash,
+        block_index: 0,
         transaction_index: transaction.index
       )
 
@@ -3002,6 +3178,8 @@ defmodule Explorer.ChainTest do
         created_contract_address: created_contract_address,
         created_contract_code: smart_contract_bytecode,
         block_number: transaction.block_number,
+        block_hash: transaction.block_hash,
+        block_index: 0,
         transaction_index: transaction.index
       )
 
@@ -3208,6 +3386,8 @@ defmodule Explorer.ChainTest do
         index: 0,
         transaction: transaction,
         block_number: transaction.block_number,
+        block_hash: transaction.block_hash,
+        block_index: 0,
         transaction_index: transaction.index
       )
 
@@ -3254,6 +3434,8 @@ defmodule Explorer.ChainTest do
         index: 0,
         transaction: transaction,
         block_number: transaction.block_number,
+        block_hash: transaction.block_hash,
+        block_index: 0,
         transaction_index: transaction.index
       )
 
@@ -3294,6 +3476,8 @@ defmodule Explorer.ChainTest do
         index: 0,
         transaction: transaction,
         block_number: transaction.block_number,
+        block_hash: transaction.block_hash,
+        block_index: 0,
         transaction_index: transaction.index
       )
 
@@ -3363,6 +3547,8 @@ defmodule Explorer.ChainTest do
         index: 0,
         transaction: from_internal_transaction_transaction,
         block_number: from_internal_transaction_transaction.block_number,
+        block_hash: from_internal_transaction_transaction.block_hash,
+        block_index: 0,
         transaction_index: from_internal_transaction_transaction.index
       )
 
@@ -3381,6 +3567,8 @@ defmodule Explorer.ChainTest do
         to_address: miner,
         transaction: to_internal_transaction_transaction,
         block_number: to_internal_transaction_transaction.block_number,
+        block_hash: to_internal_transaction_transaction.block_hash,
+        block_index: 0,
         transaction_index: to_internal_transaction_transaction.index
       )
 
@@ -3437,6 +3625,8 @@ defmodule Explorer.ChainTest do
         index: 0,
         transaction: from_internal_transaction_transaction,
         block_number: from_internal_transaction_transaction.block_number,
+        block_hash: from_internal_transaction_transaction.block_hash,
+        block_index: 0,
         transaction_index: from_internal_transaction_transaction.index
       )
 
@@ -3451,6 +3641,8 @@ defmodule Explorer.ChainTest do
         index: 0,
         transaction: to_internal_transaction_transaction,
         block_number: to_internal_transaction_transaction.block_number,
+        block_hash: to_internal_transaction_transaction.block_hash,
+        block_index: 1,
         transaction_index: to_internal_transaction_transaction.index
       )
 
@@ -4190,6 +4382,8 @@ defmodule Explorer.ChainTest do
         index: 0,
         created_contract_address: created_contract_address,
         block_number: transaction.block_number,
+        block_hash: transaction.block_hash,
+        block_index: 0,
         transaction_index: transaction.index,
         input: input
       )

apps/explorer/test/explorer/counters/average_block_time_test.exs

@@ -34,11 +34,19 @@ defmodule Explorer.Counters.AverageBlockTimeTest do
 
       first_timestamp = Timex.now()
 
-      insert(:block, number: block_number, consensus: true, timestamp: Timex.shift(first_timestamp, seconds: 3))
+      insert(:block, number: block_number, consensus: true, timestamp: Timex.shift(first_timestamp, seconds: -100 - 3))
-      insert(:block, number: block_number, consensus: false, timestamp: Timex.shift(first_timestamp, seconds: 9))
+      insert(:block, number: block_number, consensus: false, timestamp: Timex.shift(first_timestamp, seconds: -100 - 9))
-      insert(:block, number: block_number, consensus: false, timestamp: Timex.shift(first_timestamp, seconds: 6))
+      insert(:block, number: block_number, consensus: false, timestamp: Timex.shift(first_timestamp, seconds: -100 - 6))
+
+      Enum.each(1..100, fn i ->
+        insert(:block,
+          number: block_number + i,
+          consensus: true,
+          timestamp: Timex.shift(first_timestamp, seconds: -(101 - i) - 9)
+        )
+      end)
 
-      assert Repo.aggregate(Block, :count, :hash) == 3
+      assert Repo.aggregate(Block, :count, :hash) == 103
 
       AverageBlockTime.refresh()
 
@@ -55,6 +63,14 @@ defmodule Explorer.Counters.AverageBlockTimeTest do
       insert(:block, number: block_number, consensus: false, timestamp: Timex.shift(first_timestamp, seconds: 4))
       insert(:block, number: block_number + 1, consensus: true, timestamp: Timex.shift(first_timestamp, seconds: 5))
 
+      Enum.each(1..100, fn i ->
+        insert(:block,
+          number: block_number + i + 1,
+          consensus: true,
+          timestamp: Timex.shift(first_timestamp, seconds: -(101 - i) - 5)
+        )
+      end)
+
       AverageBlockTime.refresh()
 
       assert AverageBlockTime.average_block_time() == Timex.Duration.parse!("PT2S")
@@ -69,6 +85,14 @@ defmodule Explorer.Counters.AverageBlockTimeTest do
       insert(:block, number: block_number, consensus: false, timestamp: Timex.shift(first_timestamp, seconds: 4))
       insert(:block, number: block_number + 1, consensus: true, timestamp: Timex.shift(first_timestamp, seconds: 5))
 
+      Enum.each(1..100, fn i ->
+        insert(:block,
+          number: block_number + i + 1,
+          consensus: true,
+          timestamp: Timex.shift(first_timestamp, seconds: -(101 - i) - 5)
+        )
+      end)
+
       AverageBlockTime.refresh()
 
       assert AverageBlockTime.average_block_time() == Timex.Duration.parse!("PT1S")
@@ -83,7 +107,15 @@ defmodule Explorer.Counters.AverageBlockTimeTest do
       insert(:block, number: block_number + 2, consensus: true, timestamp: Timex.shift(first_timestamp, seconds: 9))
       insert(:block, number: block_number + 1, consensus: true, timestamp: Timex.shift(first_timestamp, seconds: 6))
 
-      assert Repo.aggregate(Block, :count, :hash) == 3
+      Enum.each(1..100, fn i ->
+        insert(:block,
+          number: block_number + i + 2,
+          consensus: true,
+          timestamp: Timex.shift(first_timestamp, seconds: -(101 - i) - 9)
+        )
+      end)
+
+      assert Repo.aggregate(Block, :count, :hash) == 103
 
       AverageBlockTime.refresh()
 

apps/explorer/test/explorer/etherscan_test.exs

@@ -68,6 +68,8 @@ defmodule Explorer.EtherscanTest do
           transaction: transaction,
           index: 0,
           block_number: transaction.block_number,
+          block_hash: transaction.block_hash,
+          block_index: 0,
           transaction_index: transaction.index
         )
         |> with_contract_creation(contract_address)
@@ -144,6 +146,8 @@ defmodule Explorer.EtherscanTest do
           transaction: transaction,
           index: 0,
           block_number: transaction.block_number,
+          block_hash: transaction.block_hash,
+          block_index: 0,
           transaction_index: transaction.index
         )
         |> with_contract_creation(contract_address)
@@ -485,6 +489,8 @@ defmodule Explorer.EtherscanTest do
           index: 0,
           from_address: address,
           block_number: transaction.block_number,
+          block_hash: transaction.block_hash,
+          block_index: 0,
           transaction_index: transaction.index
         )
         |> with_contract_creation(contract_address)
@@ -527,6 +533,8 @@ defmodule Explorer.EtherscanTest do
           transaction: transaction,
           index: index,
           block_number: transaction.block_number,
+          block_hash: transaction.block_hash,
+          block_index: index,
           transaction_index: transaction.index
         )
       end
@@ -551,6 +559,8 @@ defmodule Explorer.EtherscanTest do
         transaction: transaction1,
         index: 0,
         block_number: transaction1.block_number,
+        block_hash: transaction1.block_hash,
+        block_index: 0,
         transaction_index: transaction1.index
       )
 
@@ -558,6 +568,8 @@ defmodule Explorer.EtherscanTest do
         transaction: transaction1,
         index: 1,
         block_number: transaction1.block_number,
+        block_hash: transaction1.block_hash,
+        block_index: 1,
         transaction_index: transaction1.index
       )
 
@@ -566,6 +578,8 @@ defmodule Explorer.EtherscanTest do
         index: 0,
         type: :reward,
         block_number: transaction2.block_number,
+        block_hash: transaction2.block_hash,
+        block_index: 2,
         transaction_index: transaction2.index
       )
 
@@ -617,6 +631,8 @@ defmodule Explorer.EtherscanTest do
           index: 0,
           from_address: address,
           block_number: transaction.block_number,
+          block_hash: block.hash,
+          block_index: 0,
           transaction_index: transaction.index
         )
         |> with_contract_creation(contract_address)
@@ -665,6 +681,8 @@ defmodule Explorer.EtherscanTest do
           index: index,
           from_address: address,
           block_number: transaction.block_number,
+          block_hash: transaction.block_hash,
+          block_index: index,
           transaction_index: transaction.index
         }
 
@@ -689,6 +707,8 @@ defmodule Explorer.EtherscanTest do
         transaction: transaction,
         index: 0,
         block_number: transaction.block_number,
+        block_hash: transaction.block_hash,
+        block_index: 0,
         transaction_index: transaction.index,
         created_contract_address: address1
       )
@@ -697,6 +717,8 @@ defmodule Explorer.EtherscanTest do
         transaction: transaction,
         index: 1,
         block_number: transaction.block_number,
+        block_hash: transaction.block_hash,
+        block_index: 1,
         transaction_index: transaction.index,
         from_address: address1
       )
@@ -705,6 +727,8 @@ defmodule Explorer.EtherscanTest do
         transaction: transaction,
         index: 2,
         block_number: transaction.block_number,
+        block_hash: transaction.block_hash,
+        block_index: 2,
         transaction_index: transaction.index,
         to_address: address1
       )
@@ -713,6 +737,8 @@ defmodule Explorer.EtherscanTest do
         transaction: transaction,
         index: 3,
         block_number: transaction.block_number,
+        block_hash: transaction.block_hash,
+        block_index: 3,
         transaction_index: transaction.index,
         from_address: address2
       )
@@ -740,6 +766,8 @@ defmodule Explorer.EtherscanTest do
           index: index,
           from_address: address,
           block_number: transaction.block_number,
+          block_hash: transaction.block_hash,
+          block_index: index,
           transaction_index: transaction.index
         }
 
@@ -780,6 +808,8 @@ defmodule Explorer.EtherscanTest do
           index: index,
           from_address: address,
           block_number: transaction.block_number,
+          block_hash: transaction.block_hash,
+          block_index: index,
           transaction_index: transaction.index
         }
 
@@ -828,7 +858,12 @@ defmodule Explorer.EtherscanTest do
         |> insert()
         |> with_block()
 
-      token_transfer = insert(:token_transfer, transaction: transaction)
+      token_transfer =
+        insert(:token_transfer,
+          transaction: transaction,
+          block: transaction.block,
+          block_number: transaction.block_number
+        )
 
       [found_token_transfer] = Etherscan.list_token_transfers(token_transfer.from_address_hash, nil)
 
@@ -841,7 +876,12 @@ defmodule Explorer.EtherscanTest do
         |> insert()
         |> with_block()
 
-      token_transfer = insert(:token_transfer, transaction: transaction)
+      token_transfer =
+        insert(:token_transfer,
+          transaction: transaction,
+          block: transaction.block,
+          block_number: transaction.block_number
+        )
 
       [found_token_transfer] = Etherscan.list_token_transfers(token_transfer.to_address_hash, nil)
 
@@ -863,9 +903,26 @@ defmodule Explorer.EtherscanTest do
         |> insert()
         |> with_block()
 
-      insert(:token_transfer, from_address: address1, transaction: transaction)
+      insert(:token_transfer,
-      insert(:token_transfer, from_address: address1, transaction: transaction)
+        from_address: address1,
-      insert(:token_transfer, from_address: address2, transaction: transaction)
+        transaction: transaction,
+        block: transaction.block,
+        block_number: transaction.block_number
+      )
+
+      insert(:token_transfer,
+        from_address: address1,
+        transaction: transaction,
+        block: transaction.block,
+        block_number: transaction.block_number
+      )
+
+      insert(:token_transfer,
+        from_address: address2,
+        transaction: transaction,
+        block: transaction.block,
+        block_number: transaction.block_number
+      )
 
       found_token_transfers = Etherscan.list_token_transfers(address1.hash, nil)
 
@@ -884,7 +941,12 @@ defmodule Explorer.EtherscanTest do
         |> insert()
         |> with_block()
 
-      token_transfer = insert(:token_transfer, transaction: transaction)
+      token_transfer =
+        insert(:token_transfer,
+          transaction: transaction,
+          block: transaction.block,
+          block_number: transaction.block_number
+        )
 
       insert(:block)
 
@@ -903,7 +965,12 @@ defmodule Explorer.EtherscanTest do
         |> insert()
         |> with_block()
 
-      token_transfer = insert(:token_transfer, transaction: transaction)
+      token_transfer =
+        insert(:token_transfer,
+          transaction: transaction,
+          block: transaction.block,
+          block_number: transaction.block_number
+        )
 
       {:ok, token} = Chain.token_from_address_hash(token_transfer.token_contract_address_hash)
 
@@ -1019,11 +1086,29 @@ defmodule Explorer.EtherscanTest do
         |> insert()
         |> with_block(third_block)
 
-      second_block_token_transfers = insert_list(2, :token_transfer, from_address: address, transaction: transaction2)
+      second_block_token_transfers =
+        insert_list(2, :token_transfer,
+          from_address: address,
+          transaction: transaction2,
+          block: transaction2.block,
+          block_number: transaction2.block_number
+        )
 
-      first_block_token_transfers = insert_list(2, :token_transfer, from_address: address, transaction: transaction3)
+      first_block_token_transfers =
+        insert_list(2, :token_transfer,
+          from_address: address,
+          transaction: transaction3,
+          block: transaction3.block,
+          block_number: transaction3.block_number
+        )
 
-      third_block_token_transfers = insert_list(2, :token_transfer, from_address: address, transaction: transaction1)
+      third_block_token_transfers =
+        insert_list(2, :token_transfer,
+          from_address: address,
+          transaction: transaction1,
+          block: transaction1.block,
+          block_number: transaction1.block_number
+        )
 
       options1 = %{page_number: 1, page_size: 2}
 
@@ -1076,7 +1161,12 @@ defmodule Explorer.EtherscanTest do
           |> insert()
           |> with_block(block)
 
-        insert(:token_transfer, from_address: address, transaction: transaction)
+        insert(:token_transfer,
+          from_address: address,
+          transaction: transaction,
+          block: block,
+          block_number: block.number
+        )
       end
 
       options = %{
@@ -1105,7 +1195,12 @@ defmodule Explorer.EtherscanTest do
           |> insert()
           |> with_block(block)
 
-        insert(:token_transfer, from_address: address, transaction: transaction)
+        insert(:token_transfer,
+          from_address: address,
+          transaction: transaction,
+          block: block,
+          block_number: block.number
+        )
       end
 
       options = %{start_block: third_block.number}
@@ -1131,7 +1226,12 @@ defmodule Explorer.EtherscanTest do
           |> insert()
           |> with_block(block)
 
-        insert(:token_transfer, from_address: address, transaction: transaction)
+        insert(:token_transfer,
+          from_address: address,
+          transaction: transaction,
+          block: block,
+          block_number: block.number
+        )
       end
 
       options = %{end_block: second_block.number}
@@ -1160,7 +1260,14 @@ defmodule Explorer.EtherscanTest do
         |> with_block()
 
       insert(:token_transfer, from_address: address, transaction: transaction)
-      insert(:token_transfer, from_address: address, token_contract_address: contract_address, transaction: transaction)
+
+      insert(:token_transfer,
+        from_address: address,
+        token_contract_address: contract_address,
+        transaction: transaction,
+        block: transaction.block,
+        block_number: transaction.block_number
+      )
 
       [found_token_transfer] = Etherscan.list_token_transfers(address.hash, contract_address.hash)
 

apps/explorer/test/explorer/graphql_test.exs

@@ -92,9 +92,15 @@ defmodule Explorer.GraphQLTest do
 
   describe "get_internal_transaction/1" do
     test "returns existing internal transaction" do
-      transaction = insert(:transaction)
+      transaction = insert(:transaction) |> with_block()
 
-      internal_transaction = insert(:internal_transaction, transaction: transaction, index: 0)
+      internal_transaction =
+        insert(:internal_transaction,
+          transaction: transaction,
+          index: 0,
+          block_hash: transaction.block_hash,
+          block_index: 0
+        )
 
       clauses = %{transaction_hash: transaction.hash, index: internal_transaction.index}
 
@@ -117,11 +123,23 @@ defmodule Explorer.GraphQLTest do
 
   describe "transcation_to_internal_transactions_query/1" do
     test "with transaction with one internal transaction" do
-      transaction1 = insert(:transaction)
+      transaction1 = insert(:transaction) |> with_block()
-      transaction2 = insert(:transaction)
+      transaction2 = insert(:transaction) |> with_block()
-
+
-      internal_transaction = insert(:internal_transaction_create, transaction: transaction1, index: 0)
+      internal_transaction =
-      insert(:internal_transaction_create, transaction: transaction2, index: 0)
+        insert(:internal_transaction_create,
+          transaction: transaction1,
+          index: 0,
+          block_hash: transaction1.block_hash,
+          block_index: 0
+        )
+
+      insert(:internal_transaction_create,
+        transaction: transaction2,
+        index: 0,
+        block_hash: transaction2.block_hash,
+        block_index: 0
+      )
 
       [found_internal_transaction] =
         transaction1
@@ -133,14 +151,24 @@ defmodule Explorer.GraphQLTest do
     end
 
     test "with transaction with multiple internal transactions" do
-      transaction1 = insert(:transaction)
+      transaction1 = insert(:transaction) |> with_block()
-      transaction2 = insert(:transaction)
+      transaction2 = insert(:transaction) |> with_block()
 
       for index <- 0..2 do
-        insert(:internal_transaction_create, transaction: transaction1, index: index)
+        insert(:internal_transaction_create,
+          transaction: transaction1,
+          index: index,
+          block_hash: transaction1.block_hash,
+          block_index: index
+        )
       end
 
-      insert(:internal_transaction_create, transaction: transaction2, index: 0)
+      insert(:internal_transaction_create,
+        transaction: transaction2,
+        index: 0,
+        block_hash: transaction2.block_hash,
+        block_index: 0
+      )
 
       found_internal_transactions =
         transaction1
@@ -155,11 +183,28 @@ defmodule Explorer.GraphQLTest do
     end
 
     test "orders internal transactions by ascending index" do
-      transaction = insert(:transaction)
+      transaction = insert(:transaction) |> with_block()
-
+
-      insert(:internal_transaction_create, transaction: transaction, index: 2)
+      insert(:internal_transaction_create,
-      insert(:internal_transaction_create, transaction: transaction, index: 0)
+        transaction: transaction,
-      insert(:internal_transaction_create, transaction: transaction, index: 1)
+        index: 2,
+        block_hash: transaction.block_hash,
+        block_index: 2
+      )
+
+      insert(:internal_transaction_create,
+        transaction: transaction,
+        index: 0,
+        block_hash: transaction.block_hash,
+        block_index: 0
+      )
+
+      insert(:internal_transaction_create,
+        transaction: transaction,
+        index: 1,
+        block_hash: transaction.block_hash,
+        block_index: 1
+      )
 
       found_internal_transactions =
         transaction

apps/explorer/test/explorer/repo_test.exs

@@ -10,7 +10,7 @@ defmodule Explorer.RepoTest do
 
   describe "safe_insert_all/3" do
     test "inserting duplicate rows in one chunk is logged before re-raising exception" do
-      transaction = insert(:transaction)
+      transaction = insert(:transaction) |> with_block()
 
       params =
         params_for(
@@ -20,6 +20,8 @@ defmodule Explorer.RepoTest do
           transaction_hash: transaction.hash,
           index: 0,
           block_number: 35,
+          block_hash: transaction.block_hash,
+          block_index: 0,
           transaction_index: 0
         )
 
@@ -33,7 +35,7 @@ defmodule Explorer.RepoTest do
             Repo.safe_insert_all(
               InternalTransaction,
               [timestamped_changes, timestamped_changes],
-              conflict_target: [:transaction_hash, :index],
+              conflict_target: [:block_hash, :block_index],
               on_conflict: :replace_all
             )
           end
@@ -42,7 +44,7 @@ defmodule Explorer.RepoTest do
       assert log =~ "Chunk:\n"
       assert log =~ "index: 0"
 
-      assert log =~ "Options:\n\n[conflict_target: [:transaction_hash, :index], on_conflict: :replace_all]\n\n"
+      assert log =~ "Options:\n\n[conflict_target: [:block_hash, :block_index], on_conflict: :replace_all]\n\n"
 
       assert log =~
                "Exception:\n\n** (Postgrex.Error) ERROR 21000 (cardinality_violation) ON CONFLICT DO UPDATE command cannot affect row a second time\n"

apps/explorer/test/support/factory.ex

@@ -23,6 +23,7 @@ defmodule Explorer.Factory do
     Hash,
     InternalTransaction,
     Log,
+    PendingBlockOperation,
     SmartContract,
     Token,
     TokenTransfer,
@@ -226,25 +227,20 @@ defmodule Explorer.Factory do
 
     cumulative_gas_used = collated_params[:cumulative_gas_used] || Enum.random(21_000..100_000)
     gas_used = collated_params[:gas_used] || Enum.random(21_000..100_000)
-    internal_transactions_indexed_at = collated_params[:internal_transactions_indexed_at]
     status = Keyword.get(collated_params, :status, Enum.random([:ok, :error]))
 
-    error =
+    error = (status == :error && collated_params[:error]) || nil
-      if internal_transactions_indexed_at != nil && status == :error do
-        collated_params[:error] || "Something really bad happened"
-      else
-        nil
-      end
 
     transaction
     |> Transaction.changeset(%{
       block_hash: block_hash,
       block_number: block_number,
       cumulative_gas_used: cumulative_gas_used,
+      from_address_hash: transaction.from_address_hash,
+      to_address_hash: transaction.to_address_hash,
       error: error,
       gas_used: gas_used,
       index: next_transaction_index,
-      internal_transactions_indexed_at: internal_transactions_indexed_at,
       status: status
     })
     |> Repo.update!()
@@ -290,6 +286,14 @@ defmodule Explorer.Factory do
     data
   end
 
+  def pending_block_operation_factory do
+    %PendingBlockOperation{
+      # caller MUST supply block
+      # all operations will default to false
+      fetch_internal_transactions: false
+    }
+  end
+
   def internal_transaction_factory() do
     gas = Enum.random(21_000..100_000)
     gas_used = Enum.random(0..gas)
@@ -306,6 +310,8 @@ defmodule Explorer.Factory do
       trace_address: [],
       # caller MUST supply `transaction` because it can't be built lazily to allow overrides without creating an extra
       # transaction
+      # caller MUST supply `block_hash` (usually the same as the transaction's)
+      # caller MUST supply `block_index`
       type: :call,
       value: sequence("internal_transaction_value", &Decimal.new(&1))
     }
@@ -328,6 +334,8 @@ defmodule Explorer.Factory do
       trace_address: [],
       # caller MUST supply `transaction` because it can't be built lazily to allow overrides without creating an extra
       # transaction
+      # caller MUST supply `block_hash` (usually the same as the transaction's)
+      # caller MUST supply `block_index`
       type: :create,
       value: sequence("internal_transaction_value", &Decimal.new(&1))
     }
@@ -345,8 +353,12 @@ defmodule Explorer.Factory do
   end
 
   def log_factory do
+    block = build(:block)
+
     %Log{
       address: build(:address),
+      block: block,
+      block_number: block.number,
       data: data(:log_data),
       first_topic: nil,
       fourth_topic: nil,
@@ -417,6 +429,7 @@ defmodule Explorer.Factory do
     insert(:token, contract_address: token_address)
 
     %TokenTransfer{
+      block: build(:block),
       amount: Decimal.new(1),
       block_number: block_number(),
       from_address: from_address,

apps/indexer/lib/indexer/block/catchup/fetcher.ex

@@ -12,7 +12,7 @@ defmodule Indexer.Block.Catchup.Fetcher do
       async_import_block_rewards: 1,
       async_import_coin_balances: 2,
       async_import_created_contract_codes: 1,
-      async_import_internal_transactions: 2,
+      async_import_internal_transactions: 1,
       async_import_replaced_transactions: 1,
       async_import_tokens: 1,
       async_import_token_balances: 1,
@@ -122,7 +122,7 @@ defmodule Indexer.Block.Catchup.Fetcher do
   @async_import_remaining_block_data_options ~w(address_hash_to_fetched_balance_block_number)a
 
   @impl Block.Fetcher
-  def import(%Block.Fetcher{json_rpc_named_arguments: json_rpc_named_arguments}, options) when is_map(options) do
+  def import(_block_fetcher, options) when is_map(options) do
     {async_import_remaining_block_data_options, options_with_block_rewards_errors} =
       Map.split(options, @async_import_remaining_block_data_options)
 
@@ -135,8 +135,7 @@ defmodule Indexer.Block.Catchup.Fetcher do
     with {:import, {:ok, imported} = ok} <- {:import, Chain.import(full_chain_import_options)} do
       async_import_remaining_block_data(
         imported,
-        Map.put(async_import_remaining_block_data_options, :block_rewards, %{errors: block_reward_errors}),
+        Map.put(async_import_remaining_block_data_options, :block_rewards, %{errors: block_reward_errors})
-        json_rpc_named_arguments
       )
 
       ok
@@ -145,13 +144,12 @@ defmodule Indexer.Block.Catchup.Fetcher do
 
   defp async_import_remaining_block_data(
          imported,
-         %{block_rewards: %{errors: block_reward_errors}} = options,
+         %{block_rewards: %{errors: block_reward_errors}} = options
-         json_rpc_named_arguments
        ) do
     async_import_block_rewards(block_reward_errors)
     async_import_coin_balances(imported, options)
     async_import_created_contract_codes(imported)
-    async_import_internal_transactions(imported, Keyword.get(json_rpc_named_arguments, :variant))
+    async_import_internal_transactions(imported)
     async_import_tokens(imported)
     async_import_token_balances(imported)
     async_import_uncles(imported)

apps/indexer/lib/indexer/block/fetcher.ex

@@ -10,7 +10,6 @@ defmodule Indexer.Block.Fetcher do
   import EthereumJSONRPC, only: [quantity_to_integer: 1]
 
   alias EthereumJSONRPC.{Blocks, FetchedBeneficiaries}
-  alias Explorer.Chain
   alias Explorer.Chain.{Address, Block, Hash, Import, Transaction}
   alias Explorer.Chain.Cache.Blocks, as: BlocksCache
   alias Explorer.Chain.Cache.{Accounts, BlockNumber, PendingTransactions, Transactions, Uncles}
@@ -71,7 +70,6 @@ defmodule Indexer.Block.Fetcher do
 
   @receipts_batch_size 250
   @receipts_concurrency 10
-  @geth_block_limit 128
 
   @doc false
   def default_receipts_batch_size, do: @receipts_batch_size
@@ -264,14 +262,10 @@ defmodule Indexer.Block.Fetcher do
         block_number: block_number,
         hash: hash,
         created_contract_address_hash: %Hash{} = created_contract_address_hash,
-        created_contract_code_indexed_at: nil,
+        created_contract_code_indexed_at: nil
-        internal_transactions_indexed_at: nil
       } ->
         [%{block_number: block_number, hash: hash, created_contract_address_hash: created_contract_address_hash}]
 
-      %Transaction{internal_transactions_indexed_at: %DateTime{}} ->
-        []
-
       %Transaction{created_contract_address_hash: nil} ->
         []
     end)
@@ -280,30 +274,13 @@ defmodule Indexer.Block.Fetcher do
 
   def async_import_created_contract_codes(_), do: :ok
 
-  def async_import_internal_transactions(%{blocks: blocks}, EthereumJSONRPC.Parity) do
+  def async_import_internal_transactions(%{blocks: blocks}) do
     blocks
-    |> Enum.map(fn %Block{number: block_number} -> %{number: block_number} end)
+    |> Enum.map(fn %Block{number: block_number} -> block_number end)
-    |> InternalTransaction.async_block_fetch(10_000)
-  end
-
-  def async_import_internal_transactions(%{transactions: transactions}, EthereumJSONRPC.Geth) do
-    max_block_number = Chain.fetch_max_block_number()
-
-    transactions
-    |> Enum.flat_map(fn
-      %Transaction{block_number: block_number, index: index, hash: hash, internal_transactions_indexed_at: nil} ->
-        [%{block_number: block_number, index: index, hash: hash}]
-
-      %Transaction{internal_transactions_indexed_at: %DateTime{}} ->
-        []
-    end)
-    |> Enum.filter(fn %{block_number: block_number} ->
-      max_block_number - block_number < @geth_block_limit
-    end)
     |> InternalTransaction.async_fetch(10_000)
   end
 
-  def async_import_internal_transactions(_, _), do: :ok
+  def async_import_internal_transactions(_), do: :ok
 
   def async_import_tokens(%{tokens: tokens}) do
     tokens

apps/indexer/lib/indexer/block/realtime/fetcher.ex

@@ -15,7 +15,7 @@ defmodule Indexer.Block.Realtime.Fetcher do
     only: [
       async_import_block_rewards: 1,
       async_import_created_contract_codes: 1,
-      async_import_internal_transactions: 2,
+      async_import_internal_transactions: 1,
       async_import_replaced_transactions: 1,
       async_import_tokens: 1,
       async_import_token_balances: 1,
@@ -183,7 +183,7 @@ defmodule Indexer.Block.Realtime.Fetcher do
 
   @impl Block.Fetcher
   def import(
-        %Block.Fetcher{json_rpc_named_arguments: json_rpc_named_arguments} = block_fetcher,
+        block_fetcher,
         %{
           address_coin_balances: %{params: address_coin_balances_params},
           address_hash_to_fetched_balance_block_number: address_hash_to_block_number,
@@ -209,8 +209,7 @@ defmodule Indexer.Block.Realtime.Fetcher do
          {:import, {:ok, imported} = ok} <- {:import, Chain.import(chain_import_options)} do
       async_import_remaining_block_data(
         imported,
-        %{block_rewards: %{errors: block_reward_errors}},
+        %{block_rewards: %{errors: block_reward_errors}}
-        json_rpc_named_arguments
       )
 
       Accounts.drop(imported[:addresses])
@@ -381,12 +380,11 @@ defmodule Indexer.Block.Realtime.Fetcher do
 
   defp async_import_remaining_block_data(
          imported,
-         %{block_rewards: %{errors: block_reward_errors}},
+         %{block_rewards: %{errors: block_reward_errors}}
-         json_rpc_named_arguments
        ) do
     async_import_block_rewards(block_reward_errors)
     async_import_created_contract_codes(imported)
-    async_import_internal_transactions(imported, Keyword.get(json_rpc_named_arguments, :variant))
+    async_import_internal_transactions(imported)
     async_import_tokens(imported)
     async_import_token_balances(imported)
     async_import_token_instances(imported)

apps/indexer/lib/indexer/buffered_task.ex

@@ -8,6 +8,7 @@ defmodule Indexer.BufferedTask do
 
     * `:flush_interval` - The interval in milliseconds to flush the buffer.
     * `:max_concurrency` - The maximum number of tasks to run concurrently at any give time.
+    * `:poll` - poll for new records when all records are processed
     * `:max_batch_size` - The maximum batch passed to `c:run/2`.
     * `:memory_monitor` - The `Indexer.Memory.Monitor` `t:GenServer.server/0` to register as
       `Indexer.Memory.Monitor.shrinkable/0` with.
@@ -70,6 +71,7 @@ defmodule Indexer.BufferedTask do
             flush_interval: nil,
             max_batch_size: nil,
             max_concurrency: nil,
+            poll: false,
             metadata: [],
             current_buffer: [],
             bound_queue: %BoundQueue{},
@@ -229,6 +231,7 @@ defmodule Indexer.BufferedTask do
     state = %BufferedTask{
       callback_module: callback_module,
       callback_module_state: Keyword.fetch!(opts, :state),
+      poll: Keyword.get(opts, :poll, false),
       task_supervisor: Keyword.fetch!(opts, :task_supervisor),
       flush_interval: Keyword.fetch!(opts, :flush_interval),
       max_batch_size: Keyword.fetch!(opts, :max_batch_size),
@@ -434,7 +437,12 @@ defmodule Indexer.BufferedTask do
     end
   end
 
-  # was shrunk and out of work, get more work from `init/2`
+  # get more work from `init/2`
+  defp schedule_next(%BufferedTask{poll: true, bound_queue: %BoundQueue{size: 0}} = state) do
+    do_initial_stream(state)
+  end
+
+  # was shrunk and was out of work, get more work from `init/2`
   defp schedule_next(%BufferedTask{bound_queue: %BoundQueue{size: 0, maximum_size: maximum_size}} = state)
        when maximum_size != nil do
     Logger.info(fn ->

apps/indexer/lib/indexer/fetcher/internal_transaction.ex

@@ -13,7 +13,7 @@ defmodule Indexer.Fetcher.InternalTransaction do
   import Indexer.Block.Fetcher, only: [async_import_coin_balances: 2]
 
   alias Explorer.Chain
-  alias Explorer.Chain.{Block, Hash}
+  alias Explorer.Chain.Block
   alias Explorer.Chain.Cache.{Accounts, Blocks}
   alias Indexer.{BufferedTask, Tracer}
   alias Indexer.Transform.Addresses
@@ -26,6 +26,7 @@ defmodule Indexer.Fetcher.InternalTransaction do
     flush_interval: :timer.seconds(3),
     max_concurrency: @max_concurrency,
     max_batch_size: @max_batch_size,
+    poll: true,
     task_supervisor: Indexer.Fetcher.InternalTransaction.TaskSupervisor,
     metadata: [fetcher: :internal_transaction]
   ]
@@ -46,34 +47,9 @@ defmodule Indexer.Fetcher.InternalTransaction do
   *Note*: The internal transactions for individual transactions cannot be paginated,
   so the total number of internal transactions that could be produced is unknown.
   """
-  @spec async_fetch([%{required(:block_number) => Block.block_number(), required(:hash) => Hash.Full.t()}]) :: :ok
+  @spec async_fetch([Block.block_number()]) :: :ok
-  def async_fetch(transactions_fields, timeout \\ 5000) when is_list(transactions_fields) do
+  def async_fetch(block_numbers, timeout \\ 5000) when is_list(block_numbers) do
-    entries = Enum.map(transactions_fields, &entry/1)
+    BufferedTask.buffer(__MODULE__, block_numbers, timeout)
-
-    BufferedTask.buffer(__MODULE__, entries, timeout)
-  end
-
-  @doc """
-  Asynchronously fetches internal transactions.
-
-  ## Limiting Upstream Load
-
-  Internal transactions are an expensive upstream operation. The number of
-  results to fetch is configured by `@max_batch_size` and represents the number
-  of transaction hashes to request internal transactions in a single JSONRPC
-  request. Defaults to `#{@max_batch_size}`.
-
-  The `@max_concurrency` attribute configures the  number of concurrent requests
-  of `@max_batch_size` to allow against the JSONRPC. Defaults to `#{@max_concurrency}`.
-
-  *Note*: The internal transactions for individual transactions cannot be paginated,
-  so the total number of internal transactions that could be produced is unknown.
-  """
-  @spec async_block_fetch([%{required(:block_number) => Block.block_number()}]) :: :ok
-  def async_block_fetch(transactions_fields, timeout \\ 5000) when is_list(transactions_fields) do
-    entries = Enum.map(transactions_fields, &block_entry/1)
-
-    BufferedTask.buffer(__MODULE__, entries, timeout)
   end
 
   @doc false
@@ -95,52 +71,19 @@ defmodule Indexer.Fetcher.InternalTransaction do
   end
 
   @impl BufferedTask
-  def init(initial, reducer, json_rpc_named_arguments) do
+  def init(initial, reducer, _json_rpc_named_arguments) do
     {:ok, final} =
-      case Keyword.fetch!(json_rpc_named_arguments, :variant) do
+      Chain.stream_blocks_with_unfetched_internal_transactions(initial, fn block_number, acc ->
-        EthereumJSONRPC.Parity ->
+        reducer.(block_number, acc)
-          Chain.stream_blocks_with_unfetched_internal_transactions(
+      end)
-            [:number],
-            initial,
-            fn block_fields, acc ->
-              block_fields
-              |> block_entry()
-              |> reducer.(acc)
-            end
-          )
-
-        _ ->
-          Chain.stream_transactions_with_unfetched_internal_transactions(
-            [:block_number, :hash, :index],
-            initial,
-            fn transaction_fields, acc ->
-              transaction_fields
-              |> entry()
-              |> reducer.(acc)
-            end
-          )
-      end
 
     final
   end
 
-  defp entry(%{block_number: block_number, hash: %Hash{bytes: bytes}, index: index}) when is_integer(block_number) do
+  defp params(%{block_number: block_number, hash: hash, index: index}) when is_integer(block_number) do
-    {block_number, bytes, index}
-  end
-
-  defp params({block_number, hash_bytes, index}) when is_integer(block_number) do
-    {:ok, hash} = Hash.Full.cast(hash_bytes)
     %{block_number: block_number, hash_data: to_string(hash), transaction_index: index}
   end
 
-  defp block_entry(%{number: block_number}) when is_integer(block_number) do
-    block_number
-  end
-
-  defp block_params(block_number) when is_integer(block_number) do
-    %{number: block_number}
-  end
-
   @impl BufferedTask
   @decorate trace(
               name: "fetch",
@@ -148,52 +91,61 @@ defmodule Indexer.Fetcher.InternalTransaction do
               service: :indexer,
               tracer: Tracer
             )
-  def run(entries, json_rpc_named_arguments) do
+  def run(block_numbers, json_rpc_named_arguments) do
-    variant = Keyword.fetch!(json_rpc_named_arguments, :variant)
+    unique_numbers = Enum.uniq(block_numbers)
-
-    unique_entries = unique_entries(entries, variant)
 
-    unique_entries_count = Enum.count(unique_entries)
+    unique_numbers_count = Enum.count(unique_numbers)
-    Logger.metadata(count: unique_entries_count)
+    Logger.metadata(count: unique_numbers_count)
 
-    Logger.debug("fetching internal transactions for transactions")
+    Logger.debug("fetching internal transactions for blocks")
 
-    variant
+    json_rpc_named_arguments
+    |> Keyword.fetch!(:variant)
     |> case do
       EthereumJSONRPC.Parity ->
-        unique_entries
+        EthereumJSONRPC.fetch_block_internal_transactions(unique_numbers, json_rpc_named_arguments)
-        |> EthereumJSONRPC.fetch_block_internal_transactions(json_rpc_named_arguments)
 
       _ ->
-        unique_entries
+        fetch_block_internal_transactions_by_transactions(unique_numbers, json_rpc_named_arguments)
-        |> Enum.map(&params/1)
-        |> EthereumJSONRPC.fetch_internal_transactions(json_rpc_named_arguments)
     end
     |> case do
       {:ok, internal_transactions_params} ->
-        import_internal_transaction(internal_transactions_params, json_rpc_named_arguments, unique_entries)
+        import_internal_transaction(internal_transactions_params, unique_numbers)
 
       {:error, reason} ->
-        Logger.error(fn -> ["failed to fetch internal transactions for transactions: ", inspect(reason)] end,
+        Logger.error(fn -> ["failed to fetch internal transactions for blocks: ", inspect(reason)] end,
-          error_count: unique_entries_count
+          error_count: unique_numbers_count
         )
 
         # re-queue the de-duped entries
-        {:retry, unique_entries}
+        {:retry, unique_numbers}
 
       :ignore ->
         :ok
     end
   end
 
-  defp import_internal_transaction(internal_transactions_params, json_rpc_named_arguments, unique_entries) do
+  defp fetch_block_internal_transactions_by_transactions(unique_numbers, json_rpc_named_arguments) do
-    internal_transactions_indexed_at_blocks =
+    Enum.reduce(unique_numbers, {:ok, []}, fn
-      case Keyword.fetch!(json_rpc_named_arguments, :variant) do
+      block_number, {:ok, acc_list} ->
-        EthereumJSONRPC.Parity -> Enum.map(unique_entries, &block_params/1)
+        block_number
-        _ -> []
+        |> Chain.get_transactions_of_block_number()
-      end
+        |> Enum.map(&params(&1))
+        |> case do
+          [] -> {:ok, []}
+          transactions -> EthereumJSONRPC.fetch_internal_transactions(transactions, json_rpc_named_arguments)
+        end
+        |> case do
+          {:ok, internal_transactions} -> {:ok, internal_transactions ++ acc_list}
+          error_or_ignore -> error_or_ignore
+        end
+
+      _, error_or_ignore ->
+        error_or_ignore
+    end)
+  end
 
-    unique_entries_count = Enum.count(unique_entries)
+  defp import_internal_transaction(internal_transactions_params, unique_numbers) do
     internal_transactions_params_without_failed_creations = remove_failed_creations(internal_transactions_params)
 
     addresses_params =
@@ -206,14 +158,19 @@ defmodule Indexer.Fetcher.InternalTransaction do
         {hash, block_number}
       end)
 
+    empty_block_numbers =
+      unique_numbers
+      |> MapSet.new()
+      |> MapSet.difference(MapSet.new(internal_transactions_params_without_failed_creations, & &1.block_number))
+      |> Enum.map(&%{block_number: &1})
+
+    internal_transactions_and_empty_block_numbers =
+      internal_transactions_params_without_failed_creations ++ empty_block_numbers
+
     imports =
       Chain.import(%{
         addresses: %{params: addresses_params},
-        internal_transactions: %{params: internal_transactions_params_without_failed_creations},
+        internal_transactions: %{params: internal_transactions_and_empty_block_numbers, with: :blockless_changeset},
-        internal_transactions_indexed_at_blocks: %{
-          params: internal_transactions_indexed_at_blocks,
-          with: :number_only_changeset
-        },
         timeout: :infinity
       })
 
@@ -230,60 +187,19 @@ defmodule Indexer.Fetcher.InternalTransaction do
         Logger.error(
           fn ->
             [
-              "failed to import internal transactions for transactions: ",
+              "failed to import internal transactions for blocks: ",
               inspect(reason)
             ]
           end,
           step: step,
-          error_count: unique_entries_count
+          error_count: Enum.count(unique_numbers)
         )
 
         # re-queue the de-duped entries
-        {:retry, unique_entries}
+        {:retry, unique_numbers}
     end
   end
 
-  defp unique_entries(entries, EthereumJSONRPC.Parity), do: Enum.uniq(entries)
-
-  # Protection and improved reporting for https://github.com/poanetwork/blockscout/issues/289
-  defp unique_entries(entries, _) do
-    entries_by_hash_bytes = Enum.group_by(entries, &elem(&1, 1))
-
-    if map_size(entries_by_hash_bytes) < length(entries) do
-      {unique_entries, duplicate_entries} =
-        entries_by_hash_bytes
-        |> Map.values()
-        |> uniques_and_duplicates()
-
-      Logger.error(fn ->
-        duplicate_entries
-        |> Stream.with_index()
-        |> Enum.reduce(
-          ["Duplicate entries being used to fetch internal transactions:\n"],
-          fn {entry, index}, acc ->
-            [acc, "  ", to_string(index + 1), ". ", inspect(entry), "\n"]
-          end
-        )
-      end)
-
-      unique_entries
-    else
-      entries
-    end
-  end
-
-  defp uniques_and_duplicates(groups) do
-    Enum.reduce(groups, {[], []}, fn group, {acc_uniques, acc_duplicates} ->
-      case group do
-        [unique] ->
-          {[unique | acc_uniques], acc_duplicates}
-
-        [unique | _] = duplicates ->
-          {[unique | acc_uniques], duplicates ++ acc_duplicates}
-      end
-    end)
-  end
-
   defp remove_failed_creations(internal_transactions_params) do
     internal_transactions_params
     |> Enum.map(fn internal_transaction_params ->

apps/indexer/lib/indexer/pending_ops_cleaner.ex

@@ -0,0 +1,45 @@
+defmodule Indexer.PendingOpsCleaner do
+  @moduledoc """
+  Peiodically cleans non-consensus pending ops.
+  """
+
+  use GenServer
+
+  require Logger
+
+  alias Explorer.Chain
+
+  @interval :timer.minutes(60)
+
+  def start_link([init_opts, gen_server_opts]) do
+    start_link(init_opts, gen_server_opts)
+  end
+
+  def start_link(init_opts, gen_server_opts) do
+    GenServer.start_link(__MODULE__, init_opts, gen_server_opts)
+  end
+
+  def init(opts) do
+    interval = opts[:interval] || @interval
+
+    Process.send_after(self(), :clean_nonconsensus_pending_ops, interval)
+
+    {:ok, %{interval: interval}}
+  end
+
+  def handle_info(:clean_nonconsensus_pending_ops, %{interval: interval} = state) do
+    Logger.debug(fn -> "Cleaning non-consensus pending ops" end)
+
+    clean_nonconsensus_pending_ops()
+
+    Process.send_after(self(), :clean_nonconsensus_pending_ops, interval)
+
+    {:noreply, state}
+  end
+
+  defp clean_nonconsensus_pending_ops do
+    :ok = Chain.remove_nonconsensus_blocks_from_pending_ops()
+
+    Logger.debug(fn -> "Non-consensus pending ops are cleaned" end)
+  end
+end

apps/indexer/lib/indexer/supervisor.ex

@@ -5,7 +5,7 @@ defmodule Indexer.Supervisor do
 
   use Supervisor
 
-  alias Indexer.Block
+  alias Indexer.{Block, PendingOpsCleaner}
   alias Indexer.Block.{Catchup, Realtime}
 
   alias Indexer.Fetcher.{
@@ -129,7 +129,8 @@ defmodule Indexer.Supervisor do
         {UnclesWithoutIndex.Supervisor,
          [[json_rpc_named_arguments: json_rpc_named_arguments, memory_monitor: memory_monitor]]},
         {BlocksTransactionsMismatch.Supervisor,
-         [[json_rpc_named_arguments: json_rpc_named_arguments, memory_monitor: memory_monitor]]}
+         [[json_rpc_named_arguments: json_rpc_named_arguments, memory_monitor: memory_monitor]]},
+        {PendingOpsCleaner, [[], []]}
       ],
       strategy: :one_for_one
     )

apps/indexer/lib/indexer/transform/token_transfers.ex

@@ -40,6 +40,7 @@ defmodule Indexer.Transform.TokenTransfers do
     token_transfer = %{
       amount: Decimal.new(amount || 0),
       block_number: log.block_number,
+      block_hash: log.block_hash,
       log_index: log.index,
       from_address_hash: truncate_address_hash(log.second_topic),
       to_address_hash: truncate_address_hash(log.third_topic),
@@ -64,6 +65,7 @@ defmodule Indexer.Transform.TokenTransfers do
     token_transfer = %{
       block_number: log.block_number,
       log_index: log.index,
+      block_hash: log.block_hash,
       from_address_hash: truncate_address_hash(log.second_topic),
       to_address_hash: truncate_address_hash(log.third_topic),
       token_contract_address_hash: log.address_hash,
@@ -87,6 +89,7 @@ defmodule Indexer.Transform.TokenTransfers do
 
     token_transfer = %{
       block_number: log.block_number,
+      block_hash: log.block_hash,
       log_index: log.index,
       from_address_hash: encode_address_hash(from_address_hash),
       to_address_hash: encode_address_hash(to_address_hash),