Build a Block Importer.

lib/explorer/importers/block_importer.ex

@@ -0,0 +1,57 @@
+defmodule Explorer.BlockImporter do
+  @moduledoc "Imports a block."
+
+  import Ethereumex.HttpClient, only: [eth_get_block_by_number: 2]
+
+  alias Explorer.Block
+  alias Explorer.Repo.NewRelic, as: Repo
+  alias Explorer.Workers.ImportTransaction
+
+  def import(block_number) do
+    raw_block = download_block(block_number)
+    changes = extract_block(raw_block)
+
+    block = Repo.get_by(Block, hash: changes.hash) || %Block{}
+    block
+    |> Block.changeset(changes)
+    |> Repo.insert_or_update!
+
+    import_transactions(raw_block["transactions"])
+  end
+
+  def download_block(block_number) do
+    {:ok, block} = eth_get_block_by_number(block_number, false)
+    block
+  end
+
+  def extract_block(raw_block) do
+    %{
+      hash: raw_block["hash"],
+      number: raw_block["number"] |> decode_integer_field,
+      gas_used: raw_block["gasUsed"] |> decode_integer_field,
+      timestamp: raw_block["timestamp"] |> decode_time_field,
+      parent_hash: raw_block["parentHash"],
+      miner: raw_block["miner"],
+      difficulty: raw_block["difficulty"] |> decode_integer_field,
+      total_difficulty: raw_block["totalDifficulty"] |> decode_integer_field,
+      size: raw_block["size"] |> decode_integer_field,
+      gas_limit: raw_block["gasLimit"] |> decode_integer_field,
+      nonce: raw_block["nonce"] || "0",
+    }
+  end
+
+  def import_transactions(transactions) do
+    Enum.map(transactions, fn (transaction) ->
+      ImportTransaction.perform_later(transaction)
+    end)
+  end
+
+  def decode_integer_field(hex) do
+    {"0x", base_16} = String.split_at(hex, 2)
+    String.to_integer(base_16, 16)
+  end
+
+  def decode_time_field(field) do
+    field |> decode_integer_field |> Timex.from_unix
+  end
+end

lib/explorer/workers/import_block.ex

@@ -1,11 +1,11 @@
 defmodule Explorer.Workers.ImportBlock do
-  alias Explorer.Fetcher
+  alias Explorer.BlockImporter
 
   @moduledoc "Imports blocks by web3 conventions."
 
   @dialyzer {:nowarn_function, perform: 1}
   def perform(number) do
-    Fetcher.fetch("#{number}")
+    BlockImporter.import("#{number}")
   end
 
   @dialyzer {:nowarn_function, perform: 0}

test/explorer/importers/block_importer_test.exs

@@ -0,0 +1,117 @@
+defmodule Explorer.BlockImporterTest do
+  use Explorer.DataCase
+
+  import Mock
+
+  alias Explorer.Block
+  alias Explorer.BlockImporter
+  alias Explorer.Transaction
+  alias Explorer.Workers.ImportTransaction
+
+  describe "import/1" do
+    test "imports and saves a block to the database" do
+      use_cassette "block_importer_import_1_saves_the_block" do
+        with_mock ImportTransaction, [perform_later: fn (hash) -> insert(:transaction, hash: hash) end] do
+          BlockImporter.import("0xc4f0d")
+          block = Block |> order_by(desc: :inserted_at) |> Repo.one
+
+          assert block.hash == "0x16cb43ccfb7875c14eb3f03bdc098e4af053160544270594fa429d256cbca64e"
+        end
+      end
+    end
+
+    test "when a block with the same hash is imported it updates the block" do
+      use_cassette "block_importer_import_1_duplicate_block" do
+        with_mock ImportTransaction, [perform_later: fn (hash) -> insert(:transaction, hash: hash) end] do
+          insert(:block, hash: "0x16cb43ccfb7875c14eb3f03bdc098e4af053160544270594fa429d256cbca64e", gas_limit: 5)
+          BlockImporter.import("0xc4f0d")
+          block = Repo.get_by(Block, hash: "0x16cb43ccfb7875c14eb3f03bdc098e4af053160544270594fa429d256cbca64e")
+
+          assert block.gas_limit == 8000000
+          assert Block |> Repo.all |> Enum.count == 1
+        end
+      end
+    end
+
+    test "it enqueues workers that download each transaction" do
+      use_cassette "block_importer_import_1_download_transaction" do
+        with_mock ImportTransaction, [perform_later: fn (hash) -> insert(:transaction, hash: hash) end] do
+          BlockImporter.import("0xc533d")
+          last_transaction = Transaction |> order_by(desc: :inserted_at) |> limit(1) |> Repo.one
+
+          assert last_transaction.hash == "0xd32e645cbd9c03ef393a316766b19e3fd3b8937e5f2ec7bfc5fea4864e3c02e1"
+        end
+      end
+    end
+  end
+
+  describe "download_block/1" do
+    test "downloads the block" do
+      use_cassette "block_importer_download_block_1_downloads_the_block" do
+        raw_block = BlockImporter.download_block("0xc4f0d")
+        assert raw_block
+      end
+    end
+  end
+
+  describe "extract_block/1" do
+    test "extracts the block attributes" do
+      extracted_block = BlockImporter.extract_block(%{
+        "difficulty" => "0xfffffffffffffffffffffffffffffffe",
+        "gasLimit" => "0x02",
+        "gasUsed" => "0x19522",
+        "hash" => "bananas",
+        "miner" => "0xdb1207770e0a4258d7a4ce49ab037f92564fea85",
+        "number" => "0x7f2fb",
+        "parentHash" => "0x70029f66ea5a3b2b1ede95079d95a2ab74b649b5b17cdcf6f29b6317e7c7efa6",
+        "size" => "0x10",
+        "timestamp" => "0x12",
+        "totalDifficulty" => "0xff",
+        "nonce" => "0xfb6e1a62d119228b",
+        "transactions" => []
+      })
+
+      assert(extracted_block == %{
+        difficulty: 340282366920938463463374607431768211454,
+        gas_limit: 2,
+        gas_used: 103714,
+        hash: "bananas",
+        nonce: "0xfb6e1a62d119228b",
+        miner: "0xdb1207770e0a4258d7a4ce49ab037f92564fea85",
+        number: 520955,
+        parent_hash: "0x70029f66ea5a3b2b1ede95079d95a2ab74b649b5b17cdcf6f29b6317e7c7efa6",
+        size: 16,
+        timestamp: Timex.parse!("1970-01-01T00:00:18-00:00", "{ISO:Extended}"),
+        total_difficulty: 255,
+      })
+    end
+  end
+
+  describe "import_transactions/1" do
+    test "it enqueues workers that download each transaction" do
+      with_mock ImportTransaction, [perform_later: fn (hash) -> insert(:transaction, hash: hash) end] do
+        queue = BlockImporter.import_transactions([
+          "0x004bda8224214d277fc41be030fc55109afc662c5a87236de8990c8589f1c7b6",
+          "0x31d59a001b870543c2b34618aecfa8846f7c3e50e9e267119670b311c086db99",
+        ])
+        last_transaction = Transaction |> order_by(desc: :inserted_at) |> limit(1) |> Repo.one
+
+        assert last_transaction.hash == "0x004bda8224214d277fc41be030fc55109afc662c5a87236de8990c8589f1c7b6"
+        assert queue |> Enum.count == 2
+      end
+    end
+  end
+
+  describe "decode_integer_field/1" do
+    test "returns the integer value of a hex value" do
+      assert(BlockImporter.decode_integer_field("0x7f2fb") == 520955)
+    end
+  end
+
+  describe "decode_time_field/1" do
+    test "returns the date value of a hex value" do
+      the_seventies = Timex.parse!("1970-01-01T00:00:18-00:00", "{ISO:Extended}")
+      assert(BlockImporter.decode_time_field("0x12") == the_seventies)
+    end
+  end
+end

test/support/fixture/vcr_cassettes/block_importer_import_transactions_1_enqueues_workers.json

@@ -0,0 +1 @@
+[]