CipherLabs
/
blockscout
mirror of https://github.com/blockscout/blockscout
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add a scraper mix task

Browse Source
pull/2/head
Doc Ritezel 7 years ago
parent 19c9c664a0
commit bc972f9566
5 changed files with 135 additions and 99 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 191
      doc/dependency_decisions.yml
  2. 18
      lib/explorer/importers/block_importer.ex
  3. 23
      lib/mix/tasks/scrape.ex
  4. 1
      mix.exs
  5. 1
      mix.lock

191
doc/dependency_decisions.yml
Unescape View File

@ -31,316 +31,323 @@
:when: 2018-01-29 00:00:00.000000000 Z
- - :whitelist
- Simplified BSD
- :who:
:why:
- :who:
:why:
:versions: []
:when: 2018-01-29 01:07:28.529920000 Z
- - :whitelist
- BSD
- :who:
:why:
- :who:
:why:
:versions: []
:when: 2018-01-29 01:07:33.639610000 Z
- - :whitelist
- CC0-1.0
- :who:
:why:
- :who:
:why:
:versions: []
:when: 2018-01-29 01:03:40.487695000 Z
- - :whitelist
- Public Domain
- :who:
:why:
- :who:
:why:
:versions: []
:when: 2018-01-29 01:03:40.487695000 Z
- - :whitelist
- Unlicense
- :who:
:why:
- :who:
:why:
:versions: []
:when: 2018-01-29 01:03:40.487695000 Z
- - :whitelist
- ruby
- :who:
:why:
- :who:
:why:
:versions: []
:when: 2018-01-29 01:03:40.487695000 Z
- - :license
- ecto
- Apache 2.0
- :who:
:why:
- :who:
:why:
:versions: []
:when: 2018-01-29 00:00:00.000000000 Z
- - :license
- set_locale
- DWTFYWTPL
- :who:
:why:
- :who:
:why:
:versions: []
:when: 2018-01-29 00:50:47.514520000 Z
- - :license
- postgrex
- Apache 2.0
- :who:
:why:
- :who:
:why:
:versions: []
:when: 2018-01-29 00:51:23.470100000 Z
- - :license
- poison
- CC0-1.0
- :who:
:why:
- :who:
:why:
:versions: []
:when: 2018-01-29 00:51:52.519286000 Z
- - :license
- plug
- Apache 2.0
- :who:
:why:
- :who:
:why:
:versions: []
:when: 2018-01-29 00:52:14.254242000 Z
- - :license
- ethereumex
- MIT
- :who:
:why:
- :who:
:why:
:versions: []
:when: 2018-01-29 00:52:47.031205000 Z
- - :license
- hackney
- Apache 2.0
- :who:
:why:
- :who:
:why:
:versions: []
:when: 2018-01-29 00:53:32.763607000 Z
- - :license
- gettext
- Apache 2.0
- :who:
:why:
- :who:
:why:
:versions: []
:when: 2018-01-29 00:53:50.601029000 Z
- - :license
- gen_stage
- Apache 2.0
- :who:
:why:
- :who:
:why:
:versions: []
:when: 2018-01-29 00:54:18.188006000 Z
- - :license
- db_connection
- Apache 2.0
- :who:
:why:
- :who:
:why:
:versions: []
:when: 2018-01-29 00:54:36.494213000 Z
- - :license
- phoenix_live_reload
- MIT
- :who:
:why:
- :who:
:why:
:versions: []
:when: 2018-01-29 01:00:27.447229000 Z
- - :license
- dialyxir
- Apache 2.0
- :who:
:why:
- :who:
:why:
:versions: []
:when: 2018-01-29 01:00:51.180725000 Z
- - :license
- decimal
- Apache 2.0
- :who:
:why:
- :who:
:why:
:versions: []
:when: 2018-01-29 01:01:08.182752000 Z
- - :license
- file_system
- DWTFYWTPL
- :who:
:why:
- :who:
:why:
:versions: []
:when: 2018-01-29 01:01:35.443848000 Z
- - :license
- mime
- Apache 2.0
- :who:
:why:
- :who:
:why:
:versions: []
:when: 2018-01-29 01:01:59.697732000 Z
- - :license
- wordwrap
- MIT
- :who:
:why:
- :who:
:why:
:versions: []
:when: 2018-01-29 01:05:43.989619000 Z
- - :license
- path-is-inside
- MIT
- :who:
:why:
- :who:
:why:
:versions: []
:when: 2018-01-29 01:06:19.430415000 Z
- - :license
- true-case-path
- Apache 2.0
- :who:
:why:
- :who:
:why:
:versions: []
:when: 2018-01-29 01:06:35.683192000 Z
- - :license
- spdx-expression-parse
- MIT
- :who:
:why:
- :who:
:why:
:versions: []
:when: 2018-01-29 01:06:48.440333000 Z
- - :license
- rc
- MIT
- :who:
:why:
- :who:
:why:
:versions: []
:when: 2018-01-29 01:06:56.719276000 Z
- - :license
- amdefine
- MIT
- :who:
:why:
- :who:
:why:
:versions: []
:when: 2018-01-29 01:07:04.040453000 Z
- - :license
- normalize-scss
- MIT
- :who:
:why:
- :who:
:why:
:versions: []
:when: 2018-01-29 01:07:04.040453000 Z
- - :license
- optimist
- MIT
- :who:
:why:
- :who:
:why:
:versions: []
:when: 2018-01-29 01:07:04.040453000 Z
- - :license
- ua-parser-js
- MIT
- :who:
:why:
- :who:
:why:
:versions: []
:when: 2018-01-29 01:07:04.040453000 Z
- - :license
- indexof
- MIT
- :who:
:why:
- :who:
:why:
:versions: []
:when: 2018-01-29 01:10:35.556773000 Z
- - :license
- bundler
- MIT
- :who:
:why:
- :who:
:why:
:versions: []
:when: 2018-01-29 02:58:26.066580000 Z
- - :license
- abnf2
- MIT
- :who:
:why:
- :who:
:why:
:versions: []
:when: 2018-02-01 07:42:55.657686000 Z
- - :license
- ex_cldr
- Apache 2.0
- :who:
:why:
- :who:
:why:
:versions: []
:when: 2018-02-01 07:43:30.533305000 Z
- - :license
- ex_cldr_numbers
- Apache 2.0
- :who:
:why:
- :who:
:why:
:versions: []
:when: 2018-02-01 07:43:48.924488000 Z
- - :license
- ex_cldr_units
- Apache 2.0
- :who:
:why:
- :who:
:why:
:versions: []
:when: 2018-02-01 07:44:03.250381000 Z
- - :license
- math
- Apache 2.0
- :who:
:why:
- :who:
:why:
:versions: []
:when: 2018-02-02 09:32:25.722843000 Z
- - :license
- scrivener
- MIT
- :who:
:why:
- :who:
:why:
:versions: []
:when: 2018-02-02 09:32:41.813374000 Z
- - :license
- scrivener_ecto
- MIT
- :who:
:why:
- :who:
:why:
:versions: []
:when: 2018-02-02 09:32:51.070166000 Z
- - :license
- exq
- Apache 2.0
- :who:
:why:
- :who:
:why:
:versions: []
:when: 2018-02-05 04:51:08.335169000 Z
- - :license
- exq_ui
- Apache 2.0
- :who:
:why:
- :who:
:why:
:versions: []
:when: 2018-02-05 04:51:16.722700000 Z
- - :license
- async-foreach
- MIT
- :who:
:why:
- :who:
:why:
:versions: []
:when: 2018-02-05 04:53:27.833163000 Z
- - :license
- json-schema
- BSD
- :who:
:why:
- :who:
:why:
:versions: []
:when: 2018-02-05 04:53:34.656478000 Z
- - :whitelist
- CC-BY-4.0
- :who:
:why:
- :who:
:why:
:versions: []
:when: 2018-02-05 04:53:59.165323000 Z
- - :license
- sntp
- BSD
- :who:
:why:
- :who:
:why:
:versions: []
:when: 2018-02-05 04:54:45.159644000 Z
- - :license
- fsevents
- MIT
- :who:
:why:
- :who:
:why:
:versions: []
:when: 2018-02-05 04:53:27.833163000 Z
- - :license
- flow
- Apache 2.0
- :who:
:why:
:versions: []
:when: 2018-02-09 05:08:02.294651000 Z

18
lib/explorer/importers/block_importer.ex
Unescape View File

@ -8,6 +8,15 @@ defmodule Explorer.BlockImporter do
alias Explorer.Repo.NewRelic, as: Repo
alias Explorer.Workers.ImportTransaction
def import(raw_block) when is_map(raw_block) do
changes = extract_block(raw_block)
block = changes.hash |> find()
if is_nil(block.id), do: block |> Block.changeset(changes) |> Repo.insert
Enum.map(raw_block["transactions"], &ImportTransaction.perform/1)
end
@dialyzer {:nowarn_function, import: 1}
def import("pending") do
raw_block = download_block("pending")
@ -16,13 +25,8 @@ defmodule Explorer.BlockImporter do
@dialyzer {:nowarn_function, import: 1}
def import(block_number) do
raw_block = download_block(block_number)
changes = extract_block(raw_block)
block = changes.hash |> find()
if is_nil(block.id), do: block |> Block.changeset(changes) |> Repo.insert
Enum.map(raw_block["transactions"], &ImportTransaction.perform/1)
alias Explorer.BlockImporter
block_number |> download_block() |> BlockImporter.import()
end
def find(hash) do

23
lib/mix/tasks/scrape.ex
Unescape View File

@ -0,0 +1,23 @@
defmodule Mix.Tasks.Scrape do
@moduledoc "Scrapes blocks from web3"
use Mix.Task
alias Explorer.Repo
alias Explorer.SkippedBlocks
alias Explorer.BlockImporter
def run([]), do: run(1)
def run(count) do
[:postgrex, :ecto, :ethereumex, :tzdata]
|> Enum.each(&Application.ensure_all_started/1)
Repo.start_link()
"#{count}"
|> String.to_integer()
|> SkippedBlocks.first()
|> Enum.shuffle()
|> Flow.from_enumerable()
|> Flow.map(&BlockImporter.download_block/1)
|> Flow.map(&BlockImporter.import/1)
|> Enum.to_list()
end
end

1
mix.exs
Unescape View File

@ -61,6 +61,7 @@ defmodule Explorer.Mixfile do
{:exq, "~> 0.9.1"},
{:exq_ui, "~> 0.9.0"},
{:exvcr, "~> 0.10", only: :test},
{:flow, "~> 0.12"},
{:gettext, "~> 0.11"},
{:httpoison, "~> 1.0", override: true},
{:junit_formatter, ">= 0.0.0", only: [:test], runtime: false},

1
mix.lock
Unescape View File

@ -24,6 +24,7 @@
"exq_ui": {:hex, :exq_ui, "0.9.0", "e97e9fa9009f30d2926b51a166e40a3a521e83f61f3f333fede8335b2aa57f09", [:mix], [{:cowboy, "~> 1.0", [hex: :cowboy, repo: "hexpm", optional: false]}, {:exq, "~> 0.9", [hex: :exq, repo: "hexpm", optional: false]}, {:plug, ">= 0.8.1 and < 2.0.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"},
"exvcr": {:hex, :exvcr, "0.10.0", "5150808404d9f48dbda636f70f7f8fefd93e2433cd39f695f810e73b3a9d1736", [:mix], [{:exactor, "~> 2.2", [hex: :exactor, repo: "hexpm", optional: false]}, {:exjsx, "~> 4.0", [hex: :exjsx, repo: "hexpm", optional: false]}, {:httpoison, "~> 0.13", [hex: :httpoison, repo: "hexpm", optional: true]}, {:httpotion, "~> 3.0", [hex: :httpotion, repo: "hexpm", optional: true]}, {:ibrowse, "~> 4.4", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:meck, "~> 0.8.8", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm"},
"file_system": {:hex, :file_system, "0.2.4", "f0bdda195c0e46e987333e986452ec523aed21d784189144f647c43eaf307064", [:mix], [], "hexpm"},
"flow": {:hex, :flow, "0.12.0", "32c5a5f3ff6693e004b6c17a8c64dce2f8cdaf9564912d79427176013a586ab6", [:mix], [{:gen_stage, "~> 0.12.0", [hex: :gen_stage, repo: "hexpm", optional: false]}], "hexpm"},
"gen_stage": {:hex, :gen_stage, "0.12.2", "e0e347cbb1ceb5f4e68a526aec4d64b54ad721f0a8b30aa9d28e0ad749419cbb", [:mix], [], "hexpm"},
"gettext": {:hex, :gettext, "0.14.0", "1a019a2e51d5ad3d126efe166dcdf6563768e5d06c32a99ad2281a1fa94b4c72", [:mix], [], "hexpm"},
"hackney": {:hex, :hackney, "1.11.0", "4951ee019df102492dabba66a09e305f61919a8a183a7860236c0fde586134b6", [:rebar3], [{:certifi, "2.0.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "5.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "1.0.2", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"},

Write Preview
Loading…
Cancel
Save