diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index 5410ef9425..6bf306c0ce 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -18,7 +18,7 @@ defmodule Explorer.Chain do ] alias Ecto.Adapters.SQL - alias Ecto.Multi + alias Ecto.{Changeset, Multi} alias Explorer.Chain.{ Address, @@ -2047,12 +2047,27 @@ defmodule Explorer.Chain do token_changeset = Token.changeset(token, params) address_name_changeset = Address.Name.changeset(%Address.Name{}, Map.put(params, :address_hash, address_hash)) - token_opts = [on_conflict: Runner.Tokens.default_on_conflict(), conflict_target: :contract_address_hash] + stale_error_field = :contract_address_hash + stale_error_message = "is up to date" + + token_opts = [ + on_conflict: Runner.Tokens.default_on_conflict(), + conflict_target: :contract_address_hash, + stale_error_field: stale_error_field, + stale_error_message: stale_error_message + ] + address_name_opts = [on_conflict: :nothing, conflict_target: [:address_hash, :name]] insert_result = Multi.new() - |> Multi.insert(:token, token_changeset, token_opts) + |> Multi.run(:token, fn repo, _ -> + with {:error, %Changeset{errors: [{^stale_error_field, {^stale_error_message, []}}]}} <- + repo.insert(token_changeset, token_opts) do + # the original token passed into `update_token/2` as stale error means it is unchanged + {:ok, token} + end + end) |> Multi.run( :address_name, fn repo, _ -> diff --git a/apps/explorer/lib/explorer/chain/import/runner/address/coin_balances.ex b/apps/explorer/lib/explorer/chain/import/runner/address/coin_balances.ex index 7df1a28c82..16c3f611bc 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/address/coin_balances.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/address/coin_balances.ex @@ -93,8 +93,6 @@ defmodule Explorer.Chain.Import.Runner.Address.CoinBalances do balance in CoinBalance, update: [ set: [ - inserted_at: fragment("LEAST(EXCLUDED.inserted_at, ?)", balance.inserted_at), - updated_at: fragment("GREATEST(EXCLUDED.updated_at, ?)", balance.updated_at), value: fragment( """ @@ -120,9 +118,14 @@ defmodule Explorer.Chain.Import.Runner.Address.CoinBalances do balance.value_fetched_at, balance.value_fetched_at, balance.value_fetched_at - ) + ), + inserted_at: fragment("LEAST(EXCLUDED.inserted_at, ?)", balance.inserted_at), + updated_at: fragment("GREATEST(EXCLUDED.updated_at, ?)", balance.updated_at) ] - ] + ], + where: + fragment("EXCLUDED.value IS NOT NULL") and + (is_nil(balance.value_fetched_at) or fragment("EXCLUDED.value_fetched_at > ?", balance.value_fetched_at)) ) end end diff --git a/apps/explorer/lib/explorer/chain/import/runner/address/current_token_balances.ex b/apps/explorer/lib/explorer/chain/import/runner/address/current_token_balances.ex index e07b39abc0..1af76ea9b8 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/address/current_token_balances.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/address/current_token_balances.ex @@ -89,38 +89,14 @@ defmodule Explorer.Chain.Import.Runner.Address.CurrentTokenBalances do current_token_balance in CurrentTokenBalance, update: [ set: [ - block_number: - fragment( - "CASE WHEN EXCLUDED.block_number > ? THEN EXCLUDED.block_number ELSE ? END", - current_token_balance.block_number, - current_token_balance.block_number - ), - inserted_at: - fragment( - "CASE WHEN EXCLUDED.block_number > ? THEN EXCLUDED.inserted_at ELSE ? END", - current_token_balance.block_number, - current_token_balance.inserted_at - ), - updated_at: - fragment( - "CASE WHEN EXCLUDED.block_number > ? THEN EXCLUDED.updated_at ELSE ? END", - current_token_balance.block_number, - current_token_balance.updated_at - ), - value: - fragment( - "CASE WHEN EXCLUDED.block_number > ? THEN EXCLUDED.value ELSE ? END", - current_token_balance.block_number, - current_token_balance.value - ), - value_fetched_at: - fragment( - "CASE WHEN EXCLUDED.block_number > ? THEN EXCLUDED.value_fetched_at ELSE ? END", - current_token_balance.block_number, - current_token_balance.value_fetched_at - ) + block_number: fragment("EXCLUDED.block_number"), + value: fragment("EXCLUDED.value"), + value_fetched_at: fragment("EXCLUDED.value_fetched_at"), + inserted_at: fragment("LEAST(EXCLUDED.inserted_at, ?)", current_token_balance.inserted_at), + updated_at: fragment("GREATEST(EXCLUDED.updated_at, ?)", current_token_balance.updated_at) ] - ] + ], + where: fragment("? < EXCLUDED.block_number", current_token_balance.block_number) ) end end diff --git a/apps/explorer/lib/explorer/chain/import/runner/address/token_balances.ex b/apps/explorer/lib/explorer/chain/import/runner/address/token_balances.ex index 2a8b98e8ba..8d4d6ef754 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/address/token_balances.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/address/token_balances.ex @@ -80,36 +80,16 @@ defmodule Explorer.Chain.Import.Runner.Address.TokenBalances do token_balance in TokenBalance, update: [ set: [ + value: fragment("EXCLUDED.value"), + value_fetched_at: fragment("EXCLUDED.value_fetched_at"), inserted_at: fragment("LEAST(EXCLUDED.inserted_at, ?)", token_balance.inserted_at), - updated_at: fragment("GREATEST(EXCLUDED.updated_at, ?)", token_balance.updated_at), - value: - fragment( - """ - CASE WHEN EXCLUDED.value IS NOT NULL AND (? IS NULL OR EXCLUDED.value_fetched_at > ?) THEN - EXCLUDED.value - ELSE - ? - END - """, - token_balance.value_fetched_at, - token_balance.value_fetched_at, - token_balance.value - ), - value_fetched_at: - fragment( - """ - CASE WHEN EXCLUDED.value IS NOT NULL AND (? IS NULL OR EXCLUDED.value_fetched_at > ?) THEN - EXCLUDED.value_fetched_at - ELSE - ? - END - """, - token_balance.value_fetched_at, - token_balance.value_fetched_at, - token_balance.value_fetched_at - ) + updated_at: fragment("GREATEST(EXCLUDED.updated_at, ?)", token_balance.updated_at) ] - ] + ], + where: + fragment("EXCLUDED.value IS NOT NULL") and + (is_nil(token_balance.value_fetched_at) or + fragment("? < EXCLUDED.value_fetched_at", token_balance.value_fetched_at)) ) end end diff --git a/apps/explorer/lib/explorer/chain/import/runner/addresses.ex b/apps/explorer/lib/explorer/chain/import/runner/addresses.ex index 75681efdaa..15fbbbe695 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/addresses.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/addresses.ex @@ -101,7 +101,16 @@ defmodule Explorer.Chain.Import.Runner.Addresses do ), nonce: fragment("GREATEST(EXCLUDED.nonce, ?)", address.nonce) ] - ] + ], + # where any of `set`s would make a change + # This is so that tuples are only generated when a change would occur + where: + fragment("COALESCE(?, EXCLUDED.contract_code) IS DISTINCT FROM ?", address.contract_code, address.contract_code) or + fragment( + "EXCLUDED.fetched_coin_balance_block_number IS NOT NULL AND (? IS NULL OR EXCLUDED.fetched_coin_balance_block_number >= ?)", + address.fetched_coin_balance_block_number, + address.fetched_coin_balance_block_number + ) or fragment("GREATEST(?, EXCLUDED.nonce) IS DISTINCT FROM ?", address.nonce, address.nonce) ) end diff --git a/apps/explorer/lib/explorer/chain/import/runner/block/second_degree_relations.ex b/apps/explorer/lib/explorer/chain/import/runner/block/second_degree_relations.ex index 7c91b3b7b2..972406882b 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/block/second_degree_relations.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/block/second_degree_relations.ex @@ -77,7 +77,13 @@ defmodule Explorer.Chain.Import.Runner.Block.SecondDegreeRelations do uncle_fetched_at: fragment("LEAST(?, EXCLUDED.uncle_fetched_at)", block_second_degree_relation.uncle_fetched_at) ] - ] + ], + where: + fragment( + "LEAST(?, EXCLUDED.uncle_fetched_at) IS DISTINCT FROM ?", + block_second_degree_relation.uncle_fetched_at, + block_second_degree_relation.uncle_fetched_at + ) ) end end diff --git a/apps/explorer/lib/explorer/chain/import/runner/block_rewards.ex b/apps/explorer/lib/explorer/chain/import/runner/block_rewards.ex index e9ee2abcd6..9a4de5464d 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/block_rewards.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/block_rewards.ex @@ -3,8 +3,6 @@ defmodule Explorer.Chain.Import.Runner.Block.Rewards do Bulk imports `t:Explorer.Chain.Block.Reward.t/0`. """ - import Ecto.Query, only: [from: 2] - alias Ecto.{Changeset, Multi, Repo} alias Explorer.Chain.Block.Reward alias Explorer.Chain.Import @@ -53,27 +51,11 @@ defmodule Explorer.Chain.Import.Runner.Block.Rewards do repo, changes_list, conflict_target: [:address_hash, :address_type, :block_hash], - on_conflict: on_conflict(), + on_conflict: :nothing, for: ecto_schema_module(), returning: true, timeout: timeout, timestamps: timestamps ) end - - defp on_conflict do - from( - block_reward in Reward, - update: [ - set: [ - address_hash: block_reward.address_hash, - address_type: block_reward.address_type, - block_hash: block_reward.block_hash, - reward: block_reward.reward, - inserted_at: fragment("LEAST(?, EXCLUDED.inserted_at)", block_reward.inserted_at), - updated_at: fragment("GREATEST(?, EXCLUDED.updated_at)", block_reward.updated_at) - ] - ] - ) - end end diff --git a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex index cb174499d6..9f24d0be7b 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex @@ -205,6 +205,7 @@ defmodule Explorer.Chain.Import.Runner.Blocks do ) end + # credo:disable-for-next-line Credo.Check.Refactor.CyclomaticComplexity defp default_on_conflict do from( block in Block, @@ -225,7 +226,14 @@ defmodule Explorer.Chain.Import.Runner.Blocks do inserted_at: fragment("LEAST(?, EXCLUDED.inserted_at)", block.inserted_at), updated_at: fragment("GREATEST(?, EXCLUDED.updated_at)", block.updated_at) ] - ] + ], + where: + fragment("EXCLUDED.consensus <> ?", block.consensus) or fragment("EXCLUDED.difficulty <> ?", block.difficulty) or + fragment("EXCLUDED.gas_limit <> ?", block.gas_limit) or fragment("EXCLUDED.gas_used <> ?", block.gas_used) or + fragment("EXCLUDED.miner_hash <> ?", block.miner_hash) or fragment("EXCLUDED.nonce <> ?", block.nonce) or + fragment("EXCLUDED.number <> ?", block.number) or fragment("EXCLUDED.parent_hash <> ?", block.parent_hash) or + fragment("EXCLUDED.size <> ?", block.size) or fragment("EXCLUDED.timestamp <> ?", block.timestamp) or + fragment("EXCLUDED.total_difficulty <> ?", block.total_difficulty) ) end diff --git a/apps/explorer/lib/explorer/chain/import/runner/internal_transactions.ex b/apps/explorer/lib/explorer/chain/import/runner/internal_transactions.ex index 024800951f..08ff1f60a1 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/internal_transactions.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/internal_transactions.ex @@ -120,7 +120,27 @@ defmodule Explorer.Chain.Import.Runner.InternalTransactions do inserted_at: fragment("LEAST(?, EXCLUDED.inserted_at)", internal_transaction.inserted_at), updated_at: fragment("GREATEST(?, EXCLUDED.updated_at)", internal_transaction.updated_at) ] - ] + ], + # `IS DISTINCT FROM` is used because it allows `NULL` to be equal to itself + where: + fragment( + "(EXCLUDED.call_type, EXCLUDED.created_contract_address_hash, EXCLUDED.created_contract_code, EXCLUDED.error, EXCLUDED.from_address_hash, EXCLUDED.gas, EXCLUDED.gas_used, EXCLUDED.init, EXCLUDED.input, EXCLUDED.output, EXCLUDED.to_address_hash, EXCLUDED.trace_address, EXCLUDED.transaction_index, EXCLUDED.type, EXCLUDED.value) IS DISTINCT FROM (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + internal_transaction.call_type, + internal_transaction.created_contract_address_hash, + internal_transaction.created_contract_code, + internal_transaction.error, + internal_transaction.from_address_hash, + internal_transaction.gas, + internal_transaction.gas_used, + internal_transaction.init, + internal_transaction.input, + internal_transaction.output, + internal_transaction.to_address_hash, + internal_transaction.trace_address, + internal_transaction.transaction_index, + internal_transaction.type, + internal_transaction.value + ) ) end diff --git a/apps/explorer/lib/explorer/chain/import/runner/logs.ex b/apps/explorer/lib/explorer/chain/import/runner/logs.ex index 377182e22b..086a1cc6b2 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/logs.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/logs.ex @@ -91,7 +91,18 @@ defmodule Explorer.Chain.Import.Runner.Logs do inserted_at: fragment("LEAST(?, EXCLUDED.inserted_at)", log.inserted_at), updated_at: fragment("GREATEST(?, EXCLUDED.updated_at)", log.updated_at) ] - ] + ], + where: + fragment( + "(EXCLUDED.address_hash, EXCLUDED.data, EXCLUDED.first_topic, EXCLUDED.second_topic, EXCLUDED.third_topic, EXCLUDED.fourth_topic, EXCLUDED.type) IS DISTINCT FROM (?, ?, ?, ?, ?, ?, ?)", + log.address_hash, + log.data, + log.first_topic, + log.second_topic, + log.third_topic, + log.fourth_topic, + log.type + ) ) end end diff --git a/apps/explorer/lib/explorer/chain/import/runner/token_transfers.ex b/apps/explorer/lib/explorer/chain/import/runner/token_transfers.ex index 4521dc41a7..fb98b76dee 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/token_transfers.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/token_transfers.ex @@ -85,7 +85,16 @@ defmodule Explorer.Chain.Import.Runner.TokenTransfers do inserted_at: fragment("LEAST(?, EXCLUDED.inserted_at)", token_transfer.inserted_at), updated_at: fragment("GREATEST(?, EXCLUDED.updated_at)", token_transfer.updated_at) ] - ] + ], + where: + fragment( + "(EXCLUDED.amount, EXCLUDED.from_address_hash, EXCLUDED.to_address_hash, EXCLUDED.token_contract_address_hash, EXCLUDED.token_id) IS DISTINCT FROM (?, ? ,? , ?, ?)", + token_transfer.amount, + token_transfer.from_address_hash, + token_transfer.to_address_hash, + token_transfer.token_contract_address_hash, + token_transfer.token_id + ) ) end end diff --git a/apps/explorer/lib/explorer/chain/import/runner/tokens.ex b/apps/explorer/lib/explorer/chain/import/runner/tokens.ex index 1c7feece97..316c6d42da 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/tokens.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/tokens.ex @@ -86,7 +86,17 @@ defmodule Explorer.Chain.Import.Runner.Tokens do inserted_at: fragment("LEAST(?, EXCLUDED.inserted_at)", token.inserted_at), updated_at: fragment("GREATEST(?, EXCLUDED.updated_at)", token.updated_at) ] - ] + ], + where: + fragment( + "(EXCLUDED.name, EXCLUDED.symbol, EXCLUDED.total_supply, EXCLUDED.decimals, EXCLUDED.type, EXCLUDED.cataloged) IS DISTINCT FROM (?, ?, ?, ?, ?, ?)", + token.name, + token.symbol, + token.total_supply, + token.decimals, + token.type, + token.cataloged + ) ) end end diff --git a/apps/explorer/lib/explorer/chain/import/runner/transaction/forks.ex b/apps/explorer/lib/explorer/chain/import/runner/transaction/forks.ex index d16172098d..d005b5115b 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/transaction/forks.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/transaction/forks.ex @@ -80,7 +80,8 @@ defmodule Explorer.Chain.Import.Runner.Transaction.Forks do set: [ hash: fragment("EXCLUDED.hash") ] - ] + ], + where: fragment("EXCLUDED.hash <> ?", transaction_fork.hash) ) end end diff --git a/apps/explorer/lib/explorer/chain/import/runner/transactions.ex b/apps/explorer/lib/explorer/chain/import/runner/transactions.ex index eb7fafc8b8..147cdb86f3 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/transactions.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/transactions.ex @@ -103,7 +103,30 @@ defmodule Explorer.Chain.Import.Runner.Transactions do inserted_at: fragment("LEAST(?, EXCLUDED.inserted_at)", transaction.inserted_at), updated_at: fragment("GREATEST(?, EXCLUDED.updated_at)", transaction.updated_at) ] - ] + ], + where: + fragment( + "(EXCLUDED.block_hash, EXCLUDED.block_number, EXCLUDED.created_contract_address_hash, EXCLUDED.cumulative_gas_used, EXCLUDED.cumulative_gas_used, EXCLUDED.from_address_hash, EXCLUDED.gas, EXCLUDED.gas_price, EXCLUDED.gas_used, EXCLUDED.index, EXCLUDED.internal_transactions_indexed_at, EXCLUDED.input, EXCLUDED.nonce, EXCLUDED.r, EXCLUDED.s, EXCLUDED.status, EXCLUDED.to_address_hash, EXCLUDED.v, EXCLUDED.value) IS DISTINCT FROM (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + transaction.block_hash, + transaction.block_number, + transaction.created_contract_address_hash, + transaction.cumulative_gas_used, + transaction.cumulative_gas_used, + transaction.from_address_hash, + transaction.gas, + transaction.gas_price, + transaction.gas_used, + transaction.index, + transaction.internal_transactions_indexed_at, + transaction.input, + transaction.nonce, + transaction.r, + transaction.s, + transaction.status, + transaction.to_address_hash, + transaction.v, + transaction.value + ) ) end end