From 8135fcafbdd0c719724a70da103f861291020642 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Tue, 22 Jan 2019 12:05:45 -0600 Subject: [PATCH] Allow lists of block numbers instead of ranges for beneficiares Simplifies calling as numbers don't need to grouped into ranges when getting numbers from database. --- apps/ethereum_jsonrpc/lib/ethereum_jsonrpc.ex | 7 +++-- .../lib/ethereum_jsonrpc/parity.ex | 11 ++++--- .../lib/ethereum_jsonrpc/variant.ex | 5 ++- .../test/ethereum_jsonrpc/parity_test.exs | 13 ++++---- .../test/ethereum_jsonrpc_test.exs | 2 +- apps/indexer/lib/indexer/block/fetcher.ex | 5 ++- .../block/uncataloged_rewards/importer.ex | 31 ++++--------------- 7 files changed, 30 insertions(+), 44 deletions(-) diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc.ex index 4addc36998..9b5ea02d35 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc.ex @@ -29,6 +29,7 @@ defmodule EthereumJSONRPC do Block, Blocks, FetchedBalances, + FetchedBeneficiaries, FetchedCodes, Receipts, RequestCoordinator, @@ -229,8 +230,10 @@ defmodule EthereumJSONRPC do @doc """ Fetches block reward contract beneficiaries from variant API. """ - def fetch_beneficiaries(_first.._last = range, json_rpc_named_arguments) do - Keyword.fetch!(json_rpc_named_arguments, :variant).fetch_beneficiaries(range, json_rpc_named_arguments) + @spec fetch_beneficiaries([block_number], json_rpc_named_arguments) :: + {:ok, FetchedBeneficiaries.t()} | {:error, reason :: term} | :ignore + def fetch_beneficiaries(block_numbers, json_rpc_named_arguments) when is_list(block_numbers) do + Keyword.fetch!(json_rpc_named_arguments, :variant).fetch_beneficiaries(block_numbers, json_rpc_named_arguments) end @doc """ diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/parity.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/parity.ex index 6407b2b0ee..e6a6008fd3 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/parity.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/parity.ex @@ -11,10 +11,11 @@ defmodule EthereumJSONRPC.Parity do @behaviour EthereumJSONRPC.Variant @impl EthereumJSONRPC.Variant - def fetch_beneficiaries(_.._ = block_range, json_rpc_named_arguments) when is_list(json_rpc_named_arguments) do + def fetch_beneficiaries(block_numbers, json_rpc_named_arguments) + when is_list(block_numbers) and is_list(json_rpc_named_arguments) do id_to_params = - block_range - |> block_range_to_params_list() + block_numbers + |> block_numbers_to_params_list() |> id_to_params() with {:ok, responses} <- @@ -63,8 +64,8 @@ defmodule EthereumJSONRPC.Parity do end end - defp block_range_to_params_list(_.._ = block_range) do - Enum.map(block_range, &%{block_quantity: integer_to_quantity(&1)}) + defp block_numbers_to_params_list(block_numbers) when is_list(block_numbers) do + Enum.map(block_numbers, &%{block_quantity: integer_to_quantity(&1)}) end defp trace_replay_transaction_responses_to_internal_transactions_params(responses, id_to_params) diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/variant.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/variant.ex index 1355118a1f..afbcfba8e1 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/variant.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/variant.ex @@ -14,8 +14,7 @@ defmodule EthereumJSONRPC.Variant do @type internal_transaction_params :: map() @doc """ - Fetch the block reward contract beneficiaries for a given block - range, from the variant of the Ethereum JSONRPC API. + Fetch the block reward contract beneficiaries for a given blocks from the variant of the Ethereum JSONRPC API. For more information on block reward contracts see: https://wiki.parity.io/Block-Reward-Contract.html @@ -26,7 +25,7 @@ defmodule EthereumJSONRPC.Variant do * `{:error, reason}` - there was an error at the transport level * `:ignore` - the variant does not support fetching beneficiaries """ - @callback fetch_beneficiaries(Range.t(), EthereumJSONRPC.json_rpc_named_arguments()) :: + @callback fetch_beneficiaries([EthereumJSONRPC.block_number()], EthereumJSONRPC.json_rpc_named_arguments()) :: {:ok, FetchedBeneficiaries.t()} | {:error, reason :: term} | :ignore @doc """ diff --git a/apps/ethereum_jsonrpc/test/ethereum_jsonrpc/parity_test.exs b/apps/ethereum_jsonrpc/test/ethereum_jsonrpc/parity_test.exs index fe52efa6a6..76e8195241 100644 --- a/apps/ethereum_jsonrpc/test/ethereum_jsonrpc/parity_test.exs +++ b/apps/ethereum_jsonrpc/test/ethereum_jsonrpc/parity_test.exs @@ -291,7 +291,7 @@ defmodule EthereumJSONRPC.ParityTest do end assert {:ok, %FetchedBeneficiaries{params_set: params_set}} = - EthereumJSONRPC.Parity.fetch_beneficiaries(5_080_887..5_080_887, json_rpc_named_arguments) + EthereumJSONRPC.Parity.fetch_beneficiaries([5_080_887], json_rpc_named_arguments) assert Enum.count(params_set) == 2 @@ -365,7 +365,7 @@ defmodule EthereumJSONRPC.ParityTest do end assert {:ok, %FetchedBeneficiaries{params_set: params_set, errors: []}} = - EthereumJSONRPC.Parity.fetch_beneficiaries(5_609_295..5_609_295, json_rpc_named_arguments) + EthereumJSONRPC.Parity.fetch_beneficiaries([5_609_295], json_rpc_named_arguments) assert Enum.count(params_set) == 2 @@ -396,7 +396,7 @@ defmodule EthereumJSONRPC.ParityTest do end) assert {:ok, %FetchedBeneficiaries{params_set: params_set}} = - EthereumJSONRPC.Parity.fetch_beneficiaries(5_080_887..5_080_887, json_rpc_named_arguments) + EthereumJSONRPC.Parity.fetch_beneficiaries([5_080_887], json_rpc_named_arguments) assert Enum.empty?(params_set) end @@ -473,7 +473,7 @@ defmodule EthereumJSONRPC.ParityTest do end assert {:ok, %FetchedBeneficiaries{params_set: params_set}} = - EthereumJSONRPC.Parity.fetch_beneficiaries(5_077_429..5_077_429, json_rpc_named_arguments) + EthereumJSONRPC.Parity.fetch_beneficiaries([5_077_429], json_rpc_named_arguments) assert Enum.count(params_set) == 2 @@ -594,7 +594,7 @@ defmodule EthereumJSONRPC.ParityTest do assert {:ok, %FetchedBeneficiaries{params_set: params_set}} = EthereumJSONRPC.Parity.fetch_beneficiaries( - block_number1..block_number2, + [block_number1, block_number2], json_rpc_named_arguments ) @@ -641,8 +641,7 @@ defmodule EthereumJSONRPC.ParityTest do {:error, "oops"} end) - assert {:error, "oops"} = - EthereumJSONRPC.Parity.fetch_beneficiaries(5_080_887..5_080_887, json_rpc_named_arguments) + assert {:error, "oops"} = EthereumJSONRPC.Parity.fetch_beneficiaries([5_080_887], json_rpc_named_arguments) end end end diff --git a/apps/ethereum_jsonrpc/test/ethereum_jsonrpc_test.exs b/apps/ethereum_jsonrpc/test/ethereum_jsonrpc_test.exs index 35f0345058..54bd766028 100644 --- a/apps/ethereum_jsonrpc/test/ethereum_jsonrpc_test.exs +++ b/apps/ethereum_jsonrpc/test/ethereum_jsonrpc_test.exs @@ -241,7 +241,7 @@ defmodule EthereumJSONRPCTest do {:ok, []} end) - assert EthereumJSONRPC.fetch_beneficiaries(1..1, json_rpc_named_arguments) == + assert EthereumJSONRPC.fetch_beneficiaries([1], json_rpc_named_arguments) == {:ok, %FetchedBeneficiaries{params_set: MapSet.new(), errors: []}} end end diff --git a/apps/indexer/lib/indexer/block/fetcher.ex b/apps/indexer/lib/indexer/block/fetcher.ex index a43cf561e5..2bbee00fb7 100644 --- a/apps/indexer/lib/indexer/block/fetcher.ex +++ b/apps/indexer/lib/indexer/block/fetcher.ex @@ -202,7 +202,10 @@ defmodule Indexer.Block.Fetcher do defp fetch_beneficiaries(range, json_rpc_named_arguments) do result = - with :ignore <- EthereumJSONRPC.fetch_beneficiaries(range, json_rpc_named_arguments) do + with :ignore <- + range + |> Enum.to_list() + |> EthereumJSONRPC.fetch_beneficiaries(json_rpc_named_arguments) do {:ok, %FetchedBeneficiaries{params_set: MapSet.new()}} end diff --git a/apps/indexer/lib/indexer/block/uncataloged_rewards/importer.ex b/apps/indexer/lib/indexer/block/uncataloged_rewards/importer.ex index 5c98a9eaa3..b4a4669fff 100644 --- a/apps/indexer/lib/indexer/block/uncataloged_rewards/importer.ex +++ b/apps/indexer/lib/indexer/block/uncataloged_rewards/importer.ex @@ -16,10 +16,11 @@ defmodule Indexer.Block.UncatalogedRewards.Importer do @doc """ receives a list of blocks and tries to fetch and insert rewards for them """ - def fetch_and_import_rewards(blocks_batch) do + def fetch_and_import_rewards(blocks) when is_list(blocks) do result = - blocks_batch - |> break_into_chunks_of_block_numbers() + blocks + |> Stream.map(& &1.number) + |> Stream.chunk_every(@chunk_size) |> Enum.reduce([], fn chunk, acc -> chunk |> fetch_beneficiaries() @@ -36,11 +37,9 @@ defmodule Indexer.Block.UncatalogedRewards.Importer do e in RuntimeError -> {:error, %{exception: e}} end - defp fetch_beneficiaries(chunk) do - {chunk_start, chunk_end} = Enum.min_max(chunk) - + defp fetch_beneficiaries(block_numbers) when is_list(block_numbers) do {:ok, %FetchedBeneficiaries{params_set: result}} = - with :ignore <- EthereumJSONRPC.fetch_beneficiaries(chunk_start..chunk_end, json_rpc_named_arguments()) do + with :ignore <- EthereumJSONRPC.fetch_beneficiaries(block_numbers, json_rpc_named_arguments()) do {:ok, %FetchedBeneficiaries{params_set: MapSet.new()}} end @@ -77,24 +76,6 @@ defmodule Indexer.Block.UncatalogedRewards.Importer do end) end - defp break_into_chunks_of_block_numbers(blocks) do - Enum.chunk_while( - blocks, - [], - fn block, acc -> - if (acc == [] || hd(acc) + 1 == block.number) && length(acc) < @chunk_size do - {:cont, [block.number | acc]} - else - {:cont, acc, [block.number]} - end - end, - fn - [] -> {:cont, []} - acc -> {:cont, acc, []} - end - ) - end - defp insert_reward_group([]), do: :empty defp insert_reward_group(rewards) do