Merge branch 'master' into ag-readme-2

pull/2208/head
Andrew Gross 6 years ago committed by GitHub
commit 7e64798712
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 3
      CHANGELOG.md
  2. 11
      apps/block_scout_web/lib/block_scout_web/controllers/api/rpc/eth_controller.ex
  3. 7
      apps/block_scout_web/lib/block_scout_web/controllers/api_docs_controller.ex
  4. 16
      apps/block_scout_web/lib/block_scout_web/controllers/chain/market_history_chart_controller.ex
  5. 2
      apps/block_scout_web/lib/block_scout_web/notifier.ex
  6. 1
      apps/block_scout_web/lib/block_scout_web/router.ex
  7. 34
      apps/block_scout_web/lib/block_scout_web/templates/api_docs/eth_rpc.html.eex
  8. 5
      apps/block_scout_web/lib/block_scout_web/templates/layout/_topnav.html.eex
  9. 54
      apps/block_scout_web/priv/gettext/default.pot
  10. 56
      apps/block_scout_web/priv/gettext/en/LC_MESSAGES/default.po
  11. 6
      apps/block_scout_web/test/block_scout_web/channels/exchange_rate_channel_test.exs
  12. 4
      apps/explorer/lib/explorer/application.ex
  13. 3
      apps/explorer/lib/explorer/chain.ex
  14. 4
      apps/explorer/lib/explorer/chain/block.ex
  15. 19
      apps/explorer/lib/explorer/market/market.ex
  16. 79
      apps/explorer/lib/explorer/market/market_history_cache.ex
  17. 90
      apps/explorer/test/explorer/market/market_history_cache_test.exs
  18. 27
      apps/explorer/test/explorer/market/market_test.exs
  19. 4
      apps/explorer/test/support/data_case.ex

@ -1,6 +1,6 @@
## Current
### Features
- [#2182](https://github.com/poanetwork/blockscout/pull/2182) - add market history cache
- [#2109](https://github.com/poanetwork/blockscout/pull/2109) - use bigger updates instead of `Multi` transactions in BlocksTransactionsMismatch
- [#2075](https://github.com/poanetwork/blockscout/pull/2075) - add blocks cache
- [#2151](https://github.com/poanetwork/blockscout/pull/2151) - hide dropdown menu then other networks list is empty
@ -42,6 +42,7 @@
- [#2173](https://github.com/poanetwork/blockscout/pull/2173) - handle correctly empty transactions
- [#2174](https://github.com/poanetwork/blockscout/pull/2174) - fix reward channel joining
- [#2186](https://github.com/poanetwork/blockscout/pull/2186) - fix net version test
- [#2167](https://github.com/poanetwork/blockscout/pull/2168) - feat: document eth rpc api mimicking endpoints
### Chore
- [#2127](https://github.com/poanetwork/blockscout/pull/2127) - use previouse chromedriver version

@ -29,6 +29,8 @@ defmodule BlockScoutWeb.API.RPC.EthController do
3 => "fourth"
}
def methods, do: @methods
def eth_request(%{body_params: %{"_json" => requests}} = conn, _) when is_list(requests) do
responses = responses(requests)
@ -106,7 +108,11 @@ defmodule BlockScoutWeb.API.RPC.EthController do
end
defp render_log(log) do
topics = Enum.reject([log.first_topic, log.second_topic, log.third_topic, log.fourth_topic], &is_nil/1)
topics =
Enum.reject(
[log.first_topic, log.second_topic, log.third_topic, log.fourth_topic],
&is_nil/1
)
%{
"address" => to_string(log.address_hash),
@ -245,7 +251,8 @@ defmodule BlockScoutWeb.API.RPC.EthController do
defp to_block_numbers(from_block, to_block, max_block_number, pending_block_number) do
actual_pending_block_number = pending_block_number || max_block_number
with {:ok, from} <- to_block_number(from_block, max_block_number, actual_pending_block_number),
with {:ok, from} <-
to_block_number(from_block, max_block_number, actual_pending_block_number),
{:ok, to} <- to_block_number(to_block, max_block_number, actual_pending_block_number) do
{:ok, from, to}
end

@ -1,6 +1,7 @@
defmodule BlockScoutWeb.APIDocsController do
use BlockScoutWeb, :controller
alias BlockScoutWeb.API.RPC.EthController
alias BlockScoutWeb.Etherscan
def index(conn, _params) do
@ -8,4 +9,10 @@ defmodule BlockScoutWeb.APIDocsController do
|> assign(:documentation, Etherscan.get_documentation())
|> render("index.html")
end
def eth_rpc(conn, _params) do
conn
|> assign(:documentation, EthController.methods())
|> render("eth_rpc.html")
end
end

@ -8,18 +8,20 @@ defmodule BlockScoutWeb.Chain.MarketHistoryChartController do
with true <- ajax?(conn) do
exchange_rate = Market.get_exchange_rate(Explorer.coin()) || Token.null()
recent_market_history = Market.fetch_recent_history()
market_history_data =
30
|> Market.fetch_recent_history()
|> case do
[today | the_rest] -> [%{today | closing_price: exchange_rate.usd_value} | the_rest]
data -> data
case recent_market_history do
[today | the_rest] ->
encode_market_history_data([%{today | closing_price: exchange_rate.usd_value} | the_rest])
data ->
encode_market_history_data(data)
end
|> encode_market_history_data()
json(conn, %{
history_data: market_history_data,
supply_data: available_supply(Chain.supply_for_days(30), exchange_rate)
supply_data: available_supply(Chain.supply_for_days(), exchange_rate)
})
else
_ -> unprocessable_entity(conn)

@ -37,7 +37,7 @@ defmodule BlockScoutWeb.Notifier do
exchange_rate = Market.get_exchange_rate(Explorer.coin()) || Token.null()
market_history_data =
case Market.fetch_recent_history(30) do
case Market.fetch_recent_history() do
[today | the_rest] -> [%{today | closing_price: exchange_rate.usd_value} | the_rest]
data -> data
end

@ -247,6 +247,7 @@ defmodule BlockScoutWeb.Router do
get("/chain_blocks", ChainController, :chain_blocks, as: :chain_blocks)
get("/api_docs", APIDocsController, :index)
get("/eth_rpc_api_docs", APIDocsController, :eth_rpc)
get("/:page", PageNotFoundController, :index)
end

@ -0,0 +1,34 @@
<section class="container">
<div class="card">
<div class="card-body">
<h1 class="card-title margin-bottom-sm"><%= gettext("ETH RPC API Documentation") %></h2>
<p class="api-text-monospace" data-endpoint-url="<%= BlockScoutWeb.Endpoint.url() %>/api/eth_rpc">[ <%= gettext "Base URL:" %> <%= @conn.host %>/api/eth_rpc ]</p>
<p class="card-subtitle margin-bottom-0">
<%= gettext "This API is provided to support some rpc methods in the exact format specified for ethereum nodes, which can be found " %>
<a href="https://github.com/ethereum/wiki/wiki/JSON-RPC"><%= gettext "here." %></a>
<%= gettext "This is useful to allow sending requests to blockscout without having to change anything about the request." %>
<%= gettext "However, in general, the" %> <%= link(
gettext("custom RPC"),
to: api_docs_path(@conn, :index)
) %> <%= gettext " is recommended." %>
<%= gettext "Anything not in this list is not supported. Click on the method to be taken to the documentation for that method, and check the notes section for any potential differences." %>
</p>
</div>
</div>
<div class="card">
<div class="card-body">
<table class="table">
<tr>
<th>Supported Method</th>
<th>Notes</th>
</tr>
<%= for {method, info} <- Map.to_list(@documentation) do %>
<tr>
<td> <a href="https://github.com/ethereum/wiki/wiki/JSON-RPC#<%= method %>"> <%= method %> </a> </td>
<td> <%= Map.get(info, :notes, "N/A") %> </td>
</tr>
<% end %>
</table>
</div>
</section>

@ -74,6 +74,11 @@
class: "dropdown-item #{tab_status("api_docs", @conn.request_path)}",
to: api_docs_path(@conn, :index)
) %>
<%= link(
gettext("Eth RPC"),
class: "dropdown-item #{tab_status("api_docs", @conn.request_path)}",
to: api_docs_path(@conn, :eth_rpc)
) %>
</div>
</li>
<li class="nav-item dropdown nav-item-networks">

@ -128,6 +128,7 @@ msgid "Balance"
msgstr ""
#, elixir-format
#: lib/block_scout_web/templates/api_docs/eth_rpc.html.eex:5
#: lib/block_scout_web/templates/api_docs/index.html.eex:5
msgid "Base URL:"
msgstr ""
@ -665,8 +666,8 @@ msgstr ""
#, elixir-format
#: lib/block_scout_web/templates/address_logs/index.html.eex:14
#: lib/block_scout_web/templates/layout/_topnav.html.eex:111
#: lib/block_scout_web/templates/layout/_topnav.html.eex:128
#: lib/block_scout_web/templates/layout/_topnav.html.eex:116
#: lib/block_scout_web/templates/layout/_topnav.html.eex:133
msgid "Search"
msgstr ""
@ -1479,8 +1480,8 @@ msgid "Error: Could not determine contract creator."
msgstr ""
#, elixir-format
#: lib/block_scout_web/templates/layout/_topnav.html.eex:105
#: lib/block_scout_web/templates/layout/_topnav.html.eex:109
#: lib/block_scout_web/templates/layout/_topnav.html.eex:110
#: lib/block_scout_web/templates/layout/_topnav.html.eex:114
msgid "Search by address, token symbol name, transaction hash, or block number"
msgstr ""
@ -1702,3 +1703,48 @@ msgstr ""
#: lib/block_scout_web/templates/address_decompiled_contract/index.html.eex:27
msgid "There is no decompilded contracts for this address."
msgstr ""
#, elixir-format
#: lib/block_scout_web/templates/api_docs/eth_rpc.html.eex:14
msgid " is recommended."
msgstr ""
#, elixir-format
#: lib/block_scout_web/templates/api_docs/eth_rpc.html.eex:15
msgid "Anything not in this list is not supported. Click on the method to be taken to the documentation for that method, and check the notes section for any potential differences."
msgstr ""
#, elixir-format
#: lib/block_scout_web/templates/api_docs/eth_rpc.html.eex:4
msgid "ETH RPC API Documentation"
msgstr ""
#, elixir-format
#: lib/block_scout_web/templates/layout/_topnav.html.eex:78
msgid "Eth RPC"
msgstr ""
#, elixir-format
#: lib/block_scout_web/templates/api_docs/eth_rpc.html.eex:11
msgid "However, in general, the"
msgstr ""
#, elixir-format
#: lib/block_scout_web/templates/api_docs/eth_rpc.html.eex:7
msgid "This API is provided to support some rpc methods in the exact format specified for ethereum nodes, which can be found "
msgstr ""
#, elixir-format
#: lib/block_scout_web/templates/api_docs/eth_rpc.html.eex:10
msgid "This is useful to allow sending requests to blockscout without having to change anything about the request."
msgstr ""
#, elixir-format
#: lib/block_scout_web/templates/api_docs/eth_rpc.html.eex:12
msgid "custom RPC"
msgstr ""
#, elixir-format
#: lib/block_scout_web/templates/api_docs/eth_rpc.html.eex:9
msgid "here."
msgstr ""

@ -128,6 +128,7 @@ msgid "Balance"
msgstr ""
#, elixir-format
#: lib/block_scout_web/templates/api_docs/eth_rpc.html.eex:5
#: lib/block_scout_web/templates/api_docs/index.html.eex:5
msgid "Base URL:"
msgstr ""
@ -665,8 +666,8 @@ msgstr ""
#, elixir-format
#: lib/block_scout_web/templates/address_logs/index.html.eex:14
#: lib/block_scout_web/templates/layout/_topnav.html.eex:111
#: lib/block_scout_web/templates/layout/_topnav.html.eex:128
#: lib/block_scout_web/templates/layout/_topnav.html.eex:116
#: lib/block_scout_web/templates/layout/_topnav.html.eex:133
msgid "Search"
msgstr ""
@ -1479,8 +1480,8 @@ msgid "Error: Could not determine contract creator."
msgstr ""
#, elixir-format
#: lib/block_scout_web/templates/layout/_topnav.html.eex:105
#: lib/block_scout_web/templates/layout/_topnav.html.eex:109
#: lib/block_scout_web/templates/layout/_topnav.html.eex:110
#: lib/block_scout_web/templates/layout/_topnav.html.eex:114
msgid "Search by address, token symbol name, transaction hash, or block number"
msgstr ""
@ -1698,7 +1699,52 @@ msgstr ""
msgid " Token Transfer"
msgstr ""
#, elixir-format, fuzzy
#, elixir-format
#: lib/block_scout_web/templates/address_decompiled_contract/index.html.eex:27
msgid "There is no decompilded contracts for this address."
msgstr ""
#, elixir-format
#: lib/block_scout_web/templates/api_docs/eth_rpc.html.eex:14
msgid " is recommended."
msgstr ""
#, elixir-format
#: lib/block_scout_web/templates/api_docs/eth_rpc.html.eex:15
msgid "Anything not in this list is not supported. Click on the method to be taken to the documentation for that method, and check the notes section for any potential differences."
msgstr ""
#, elixir-format, fuzzy
#: lib/block_scout_web/templates/api_docs/eth_rpc.html.eex:4
msgid "ETH RPC API Documentation"
msgstr ""
#, elixir-format
#: lib/block_scout_web/templates/layout/_topnav.html.eex:78
msgid "Eth RPC"
msgstr ""
#, elixir-format
#: lib/block_scout_web/templates/api_docs/eth_rpc.html.eex:11
msgid "However, in general, the"
msgstr ""
#, elixir-format
#: lib/block_scout_web/templates/api_docs/eth_rpc.html.eex:7
msgid "This API is provided to support some rpc methods in the exact format specified for ethereum nodes, which can be found "
msgstr ""
#, elixir-format
#: lib/block_scout_web/templates/api_docs/eth_rpc.html.eex:10
msgid "This is useful to allow sending requests to blockscout without having to change anything about the request."
msgstr ""
#, elixir-format
#: lib/block_scout_web/templates/api_docs/eth_rpc.html.eex:12
msgid "custom RPC"
msgstr ""
#, elixir-format, fuzzy
#: lib/block_scout_web/templates/api_docs/eth_rpc.html.eex:9
msgid "here."
msgstr ""

@ -43,6 +43,8 @@ defmodule BlockScoutWeb.ExchangeRateChannelTest do
describe "new_rate" do
test "subscribed user is notified", %{token: token} do
ExchangeRates.handle_info({nil, {:ok, [token]}}, %{})
Supervisor.terminate_child(Explorer.Supervisor, {ConCache, Explorer.Market.MarketHistoryCache.cache_name()})
Supervisor.restart_child(Explorer.Supervisor, {ConCache, Explorer.Market.MarketHistoryCache.cache_name()})
topic = "exchange_rate:new_rate"
@endpoint.subscribe(topic)
@ -61,6 +63,8 @@ defmodule BlockScoutWeb.ExchangeRateChannelTest do
test "subscribed user is notified with market history", %{token: token} do
ExchangeRates.handle_info({nil, {:ok, [token]}}, %{})
Supervisor.terminate_child(Explorer.Supervisor, {ConCache, Explorer.Market.MarketHistoryCache.cache_name()})
Supervisor.restart_child(Explorer.Supervisor, {ConCache, Explorer.Market.MarketHistoryCache.cache_name()})
today = Date.utc_today()
@ -76,6 +80,8 @@ defmodule BlockScoutWeb.ExchangeRateChannelTest do
Market.bulk_insert_history(records)
Market.fetch_recent_history()
topic = "exchange_rate:new_rate"
@endpoint.subscribe(topic)

@ -7,6 +7,7 @@ defmodule Explorer.Application do
alias Explorer.Admin
alias Explorer.Chain.{BlockCountCache, BlockNumberCache, BlocksCache, NetVersionCache, TransactionCountCache}
alias Explorer.Market.MarketHistoryCache
alias Explorer.Repo.PrometheusLogger
@impl Application
@ -32,7 +33,8 @@ defmodule Explorer.Application do
{TransactionCountCache, [[], []]},
{BlockCountCache, []},
con_cache_child_spec(BlocksCache.cache_name()),
con_cache_child_spec(NetVersionCache.cache_name())
con_cache_child_spec(NetVersionCache.cache_name()),
con_cache_child_spec(MarketHistoryCache.cache_name())
]
children = base_children ++ configurable_children()

@ -52,6 +52,7 @@ defmodule Explorer.Chain do
alias Explorer.Chain.Block.{EmissionReward, Reward}
alias Explorer.Chain.Import.Runner
alias Explorer.Counters.AddressesWithBalanceCounter
alias Explorer.Market.MarketHistoryCache
alias Explorer.{PagingOptions, Repo}
alias Dataloader.Ecto, as: DataloaderEcto
@ -2613,7 +2614,7 @@ defmodule Explorer.Chain do
@doc """
Calls supply_for_days from the configured supply_module
"""
def supply_for_days(days_count), do: supply_module().supply_for_days(days_count)
def supply_for_days, do: supply_module().supply_for_days(MarketHistoryCache.recent_days_count())
@doc """
Streams a lists token contract addresses that haven't been cataloged.

@ -10,9 +10,9 @@ defmodule Explorer.Chain.Block do
alias Explorer.Chain.{Address, Gas, Hash, Transaction}
alias Explorer.Chain.Block.{Reward, SecondDegreeRelation}
@optional_attrs ~w(internal_transactions_indexed_at size refetch_needed)a
@optional_attrs ~w(internal_transactions_indexed_at size refetch_needed total_difficulty difficulty)a
@required_attrs ~w(consensus difficulty gas_limit gas_used hash miner_hash nonce number parent_hash timestamp total_difficulty)a
@required_attrs ~w(consensus gas_limit gas_used hash miner_hash nonce number parent_hash timestamp)a
@typedoc """
How much work is required to find a hash with some number of leading 0s. It is measured in hashes for PoW

@ -3,12 +3,10 @@ defmodule Explorer.Market do
Context for data related to the cryptocurrency market.
"""
import Ecto.Query
alias Explorer.Chain.Address.CurrentTokenBalance
alias Explorer.Chain.Hash
alias Explorer.ExchangeRates.Token
alias Explorer.Market.MarketHistory
alias Explorer.Market.{MarketHistory, MarketHistoryCache}
alias Explorer.{ExchangeRates, KnownTokens, Repo}
@doc """
@ -35,18 +33,9 @@ defmodule Explorer.Market do
Today's date is include as part of the day count
"""
@spec fetch_recent_history(non_neg_integer()) :: [MarketHistory.t()]
def fetch_recent_history(days) when days >= 1 do
day_diff = days * -1
query =
from(
mh in MarketHistory,
where: mh.date > date_add(^Date.utc_today(), ^day_diff, "day"),
order_by: [desc: mh.date]
)
Repo.all(query)
@spec fetch_recent_history() :: [MarketHistory.t()]
def fetch_recent_history do
MarketHistoryCache.fetch()
end
@doc false

@ -0,0 +1,79 @@
defmodule Explorer.Market.MarketHistoryCache do
@moduledoc """
Caches recent market history.
"""
import Ecto.Query, only: [from: 2]
alias Explorer.Market.MarketHistory
alias Explorer.Repo
@cache_name :market_history
@last_update_key :last_update
@history_key :history
# 6 hours
@cache_period 1_000 * 60 * 60 * 6
@recent_days 30
def fetch do
if cache_expired?() do
update_cache()
else
fetch_from_cache(@history_key)
end
end
def cache_name, do: @cache_name
def data_key, do: @history_key
def updated_at_key, do: @last_update_key
def recent_days_count, do: @recent_days
defp cache_expired? do
updated_at = fetch_from_cache(@last_update_key)
cond do
is_nil(updated_at) -> true
current_time() - updated_at > @cache_period -> true
true -> false
end
end
defp update_cache do
new_data = fetch_from_db()
put_into_cache(@last_update_key, current_time())
put_into_cache(@history_key, new_data)
new_data
end
defp fetch_from_db do
day_diff = @recent_days * -1
query =
from(
mh in MarketHistory,
where: mh.date > date_add(^Date.utc_today(), ^day_diff, "day"),
order_by: [desc: mh.date]
)
Repo.all(query)
end
defp fetch_from_cache(key) do
ConCache.get(@cache_name, key)
end
defp put_into_cache(key, value) do
ConCache.put(@cache_name, key, value)
end
defp current_time do
utc_now = DateTime.utc_now()
DateTime.to_unix(utc_now, :millisecond)
end
end

@ -0,0 +1,90 @@
defmodule Explorer.Market.MarketHistoryCacheTest do
use Explorer.DataCase
alias Explorer.Market
alias Explorer.Market.MarketHistoryCache
setup do
Supervisor.terminate_child(Explorer.Supervisor, {ConCache, MarketHistoryCache.cache_name()})
Supervisor.restart_child(Explorer.Supervisor, {ConCache, MarketHistoryCache.cache_name()})
on_exit(fn ->
Supervisor.terminate_child(Explorer.Supervisor, {ConCache, Explorer.Chain.BlocksCache.cache_name()})
Supervisor.restart_child(Explorer.Supervisor, {ConCache, Explorer.Chain.BlocksCache.cache_name()})
end)
:ok
end
describe "fetch/1" do
test "caches data on the first call" do
today = Date.utc_today()
records =
for i <- 0..29 do
%{
date: Timex.shift(today, days: i * -1),
closing_price: Decimal.new(1),
opening_price: Decimal.new(1)
}
end
Market.bulk_insert_history(records)
refute fetch_data()
assert Enum.count(MarketHistoryCache.fetch()) == 30
assert fetch_data() == records
end
test "updates cache if cache is stale" do
today = Date.utc_today()
stale_records =
for i <- 0..29 do
%{
date: Timex.shift(today, days: i * -1),
closing_price: Decimal.new(1),
opening_price: Decimal.new(1)
}
end
Market.bulk_insert_history(stale_records)
MarketHistoryCache.fetch()
stale_updated_at = fetch_updated_at()
assert fetch_data() == stale_records
ConCache.put(MarketHistoryCache.cache_name(), MarketHistoryCache.updated_at_key(), 1)
fetch_data()
assert stale_updated_at != fetch_updated_at()
end
end
defp fetch_updated_at do
ConCache.get(MarketHistoryCache.cache_name(), MarketHistoryCache.updated_at_key())
end
defp fetch_data do
MarketHistoryCache.cache_name()
|> ConCache.get(MarketHistoryCache.data_key())
|> case do
nil ->
nil
records ->
Enum.map(records, fn record ->
%{
date: record.date,
closing_price: record.closing_price,
opening_price: record.opening_price
}
end)
end
end
end

@ -1,15 +1,27 @@
defmodule Explorer.MarketTest do
use Explorer.DataCase
use Explorer.DataCase, async: false
alias Explorer.Market
alias Explorer.Market.MarketHistory
alias Explorer.Repo
setup do
Supervisor.terminate_child(Explorer.Supervisor, {ConCache, Explorer.Chain.BlocksCache.cache_name()})
Supervisor.restart_child(Explorer.Supervisor, {ConCache, Explorer.Chain.BlocksCache.cache_name()})
on_exit(fn ->
Supervisor.terminate_child(Explorer.Supervisor, {ConCache, Explorer.Chain.BlocksCache.cache_name()})
Supervisor.restart_child(Explorer.Supervisor, {ConCache, Explorer.Chain.BlocksCache.cache_name()})
end)
:ok
end
test "fetch_recent_history/1" do
today = Date.utc_today()
records =
for i <- 0..5 do
for i <- 0..29 do
%{
date: Timex.shift(today, days: i * -1),
closing_price: Decimal.new(1),
@ -19,16 +31,9 @@ defmodule Explorer.MarketTest do
Market.bulk_insert_history(records)
history = Market.fetch_recent_history(1)
assert length(history) == 1
history = Market.fetch_recent_history()
assert length(history) == 30
assert Enum.at(history, 0).date == Enum.at(records, 0).date
more_history = Market.fetch_recent_history(5)
assert length(more_history) == 5
for {history_record, index} <- Enum.with_index(more_history) do
assert history_record.date == Enum.at(records, index).date
end
end
describe "bulk_insert_history/1" do

@ -40,8 +40,8 @@ defmodule Explorer.DataCase do
end
Explorer.Chain.BlockNumberCache.setup()
Supervisor.terminate_child(Explorer.Supervisor, {ConCache, :blocks})
Supervisor.restart_child(Explorer.Supervisor, {ConCache, :blocks})
Supervisor.terminate_child(Explorer.Supervisor, {ConCache, Explorer.Chain.BlocksCache.cache_name()})
Supervisor.restart_child(Explorer.Supervisor, {ConCache, Explorer.Chain.BlocksCache.cache_name()})
:ok
end

Loading…
Cancel
Save