|
|
|
@ -3,27 +3,18 @@ defmodule Explorer.Chain.Import do |
|
|
|
Bulk importing of data into `Explorer.Repo` |
|
|
|
Bulk importing of data into `Explorer.Repo` |
|
|
|
""" |
|
|
|
""" |
|
|
|
|
|
|
|
|
|
|
|
alias Ecto.{Changeset, Multi} |
|
|
|
alias Ecto.Changeset |
|
|
|
alias Explorer.Chain.Import |
|
|
|
alias Explorer.Chain.Import |
|
|
|
alias Explorer.Repo |
|
|
|
alias Explorer.Repo |
|
|
|
|
|
|
|
|
|
|
|
# in order so that foreign keys are inserted before being referenced |
|
|
|
@stages [ |
|
|
|
@runners [ |
|
|
|
Import.Stage.Addresses, |
|
|
|
Import.Addresses, |
|
|
|
Import.Stage.AddressReferencing |
|
|
|
Import.Address.CoinBalances, |
|
|
|
|
|
|
|
Import.Blocks, |
|
|
|
|
|
|
|
Import.Block.Rewards, |
|
|
|
|
|
|
|
Import.Block.SecondDegreeRelations, |
|
|
|
|
|
|
|
Import.Transactions, |
|
|
|
|
|
|
|
Import.Transaction.Forks, |
|
|
|
|
|
|
|
Import.InternalTransactions, |
|
|
|
|
|
|
|
Import.Logs, |
|
|
|
|
|
|
|
Import.Tokens, |
|
|
|
|
|
|
|
Import.TokenTransfers, |
|
|
|
|
|
|
|
Import.Address.CurrentTokenBalances, |
|
|
|
|
|
|
|
Import.Address.TokenBalances |
|
|
|
|
|
|
|
] |
|
|
|
] |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# in order so that foreign keys are inserted before being referenced |
|
|
|
|
|
|
|
@runners Enum.flat_map(@stages, fn stage -> stage.runners() end) |
|
|
|
|
|
|
|
|
|
|
|
quoted_runner_option_value = |
|
|
|
quoted_runner_option_value = |
|
|
|
quote do |
|
|
|
quote do |
|
|
|
Import.Runner.options() |
|
|
|
Import.Runner.options() |
|
|
|
@ -129,8 +120,8 @@ defmodule Explorer.Chain.Import do |
|
|
|
def all(options) when is_map(options) do |
|
|
|
def all(options) when is_map(options) do |
|
|
|
with {:ok, runner_options_pairs} <- validate_options(options), |
|
|
|
with {:ok, runner_options_pairs} <- validate_options(options), |
|
|
|
{:ok, valid_runner_option_pairs} <- validate_runner_options_pairs(runner_options_pairs), |
|
|
|
{:ok, valid_runner_option_pairs} <- validate_runner_options_pairs(runner_options_pairs), |
|
|
|
{:ok, runner_changes_list_pairs} <- runner_changes_list_pairs(valid_runner_option_pairs), |
|
|
|
{:ok, runner_to_changes_list} <- runner_to_changes_list(valid_runner_option_pairs), |
|
|
|
{:ok, data} <- insert_runner_changes_list_pairs(runner_changes_list_pairs, options) do |
|
|
|
{:ok, data} <- insert_runner_to_changes_list(runner_to_changes_list, options) do |
|
|
|
broadcast_events(data, Map.get(options, :broadcast, false)) |
|
|
|
broadcast_events(data, Map.get(options, :broadcast, false)) |
|
|
|
{:ok, data} |
|
|
|
{:ok, data} |
|
|
|
end |
|
|
|
end |
|
|
|
@ -153,13 +144,12 @@ defmodule Explorer.Chain.Import do |
|
|
|
end) |
|
|
|
end) |
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
defp runner_changes_list_pairs(runner_options_pairs) when is_list(runner_options_pairs) do |
|
|
|
defp runner_to_changes_list(runner_options_pairs) when is_list(runner_options_pairs) do |
|
|
|
{status, reversed} = |
|
|
|
|
|
|
|
runner_options_pairs |
|
|
|
runner_options_pairs |
|
|
|
|> Stream.map(fn {runner, options} -> runner_changes_list(runner, options) end) |
|
|
|
|> Stream.map(fn {runner, options} -> runner_changes_list(runner, options) end) |
|
|
|
|> Enum.reduce({:ok, []}, fn |
|
|
|
|> Enum.reduce({:ok, %{}}, fn |
|
|
|
{:ok, runner_changes_pair}, {:ok, acc_runner_changes_pairs} -> |
|
|
|
{:ok, {runner, changes_list}}, {:ok, acc_runner_to_changes_list} -> |
|
|
|
{:ok, [runner_changes_pair | acc_runner_changes_pairs]} |
|
|
|
{:ok, Map.put(acc_runner_to_changes_list, runner, changes_list)} |
|
|
|
|
|
|
|
|
|
|
|
{:ok, _}, {:error, _} = error -> |
|
|
|
{:ok, _}, {:error, _} = error -> |
|
|
|
error |
|
|
|
error |
|
|
|
@ -170,8 +160,6 @@ defmodule Explorer.Chain.Import do |
|
|
|
{:error, runner_changesets}, {:error, acc_changesets} -> |
|
|
|
{:error, runner_changesets}, {:error, acc_changesets} -> |
|
|
|
{:error, acc_changesets ++ runner_changesets} |
|
|
|
{:error, acc_changesets ++ runner_changesets} |
|
|
|
end) |
|
|
|
end) |
|
|
|
|
|
|
|
|
|
|
|
{status, Enum.reverse(reversed)} |
|
|
|
|
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
defp runner_changes_list(runner, %{params: params} = options) do |
|
|
|
defp runner_changes_list(runner, %{params: params} = options) do |
|
|
|
@ -286,14 +274,22 @@ defmodule Explorer.Chain.Import do |
|
|
|
end |
|
|
|
end |
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
defp runner_changes_list_pairs_to_multi(runner_changes_list_pairs, options) |
|
|
|
defp runner_to_changes_list_to_multis(runner_to_changes_list, options) |
|
|
|
when is_list(runner_changes_list_pairs) and is_map(options) do |
|
|
|
when is_map(runner_to_changes_list) and is_map(options) do |
|
|
|
timestamps = timestamps() |
|
|
|
timestamps = timestamps() |
|
|
|
full_options = Map.put(options, :timestamps, timestamps) |
|
|
|
full_options = Map.put(options, :timestamps, timestamps) |
|
|
|
|
|
|
|
|
|
|
|
Enum.reduce(runner_changes_list_pairs, Multi.new(), fn {runner, changes_list}, acc -> |
|
|
|
{multis, final_runner_to_changes_list} = |
|
|
|
runner.run(acc, changes_list, full_options) |
|
|
|
Enum.flat_map_reduce(@stages, runner_to_changes_list, fn stage, remaining_runner_to_changes_list -> |
|
|
|
|
|
|
|
stage.multis(remaining_runner_to_changes_list, full_options) |
|
|
|
end) |
|
|
|
end) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
unless Enum.empty?(final_runner_to_changes_list) do |
|
|
|
|
|
|
|
raise ArgumentError, |
|
|
|
|
|
|
|
"No stages consumed the following runners: #{final_runner_to_changes_list |> Map.keys() |> inspect()}" |
|
|
|
|
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
multis |
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
def insert_changes_list(repo, changes_list, options) when is_atom(repo) and is_list(changes_list) do |
|
|
|
def insert_changes_list(repo, changes_list, options) when is_atom(repo) and is_list(changes_list) do |
|
|
|
@ -319,14 +315,29 @@ defmodule Explorer.Chain.Import do |
|
|
|
Map.merge(changes, timestamps) |
|
|
|
Map.merge(changes, timestamps) |
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
defp import_transaction(multi, options) when is_map(options) do |
|
|
|
defp insert_runner_to_changes_list(runner_to_changes_list, options) when is_map(runner_to_changes_list) do |
|
|
|
Repo.transaction(multi, timeout: Map.get(options, :timeout, @transaction_timeout)) |
|
|
|
runner_to_changes_list |
|
|
|
|
|
|
|
|> runner_to_changes_list_to_multis(options) |
|
|
|
|
|
|
|
|> logged_import(options) |
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
defp insert_runner_changes_list_pairs(runner_changes_list_pairs, options) do |
|
|
|
defp logged_import(multis, options) when is_list(multis) and is_map(options) do |
|
|
|
runner_changes_list_pairs |
|
|
|
import_id = :erlang.unique_integer([:positive]) |
|
|
|
|> runner_changes_list_pairs_to_multi(options) |
|
|
|
|
|
|
|
|> import_transaction(options) |
|
|
|
Explorer.Logger.metadata(fn -> import_transactions(multis, options) end, import_id: import_id) |
|
|
|
|
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
defp import_transactions(multis, options) when is_list(multis) and is_map(options) do |
|
|
|
|
|
|
|
Enum.reduce_while(multis, {:ok, %{}}, fn multi, {:ok, acc_changes} -> |
|
|
|
|
|
|
|
case import_transaction(multi, options) do |
|
|
|
|
|
|
|
{:ok, changes} -> {:cont, {:ok, Map.merge(acc_changes, changes)}} |
|
|
|
|
|
|
|
{:error, _, _, _} = error -> {:halt, error} |
|
|
|
|
|
|
|
end |
|
|
|
|
|
|
|
end) |
|
|
|
|
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
defp import_transaction(multi, options) when is_map(options) do |
|
|
|
|
|
|
|
Repo.logged_transaction(multi, timeout: Map.get(options, :timeout, @transaction_timeout)) |
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
@spec timestamps() :: timestamps |
|
|
|
@spec timestamps() :: timestamps |
|
|
|
|
Luke Imhoff
6 years ago
committed by
GitHub