From 32243b0c4ed4bdefc4fd7ca9321762fcf315e5a0 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Mon, 8 Jul 2019 10:43:33 +0300 Subject: [PATCH 1/3] fix market history overriding --- .../market/history/source/crypto_compare.ex | 19 ++++--- apps/explorer/lib/explorer/market/market.ex | 7 ++- .../history/source/crypto_compare_test.exs | 54 +++++++++++++++++++ .../test/explorer/market/market_test.exs | 19 +++++++ 4 files changed, 91 insertions(+), 8 deletions(-) diff --git a/apps/explorer/lib/explorer/market/history/source/crypto_compare.ex b/apps/explorer/lib/explorer/market/history/source/crypto_compare.ex index 82bf3714d5..09959f6cca 100644 --- a/apps/explorer/lib/explorer/market/history/source/crypto_compare.ex +++ b/apps/explorer/lib/explorer/market/history/source/crypto_compare.ex @@ -48,13 +48,18 @@ defmodule Explorer.Market.History.Source.CryptoCompare do defp format_data(data) do json = Jason.decode!(data) - for item <- json["Data"] do - %{ - closing_price: Decimal.new(to_string(item["close"])), - date: date(item["time"]), - opening_price: Decimal.new(to_string(item["open"])) - } - end + items = + for item <- json["Data"] do + %{ + closing_price: Decimal.new(to_string(item["close"])), + date: date(item["time"]), + opening_price: Decimal.new(to_string(item["open"])) + } + end + + Enum.reject(items, fn item -> + Decimal.equal?(item.closing_price, 0) && Decimal.equal?(item.opening_price, 0) + end) end @spec history_url(non_neg_integer()) :: String.t() diff --git a/apps/explorer/lib/explorer/market/market.ex b/apps/explorer/lib/explorer/market/market.ex index f3e35f7d64..a9be83a5c8 100644 --- a/apps/explorer/lib/explorer/market/market.ex +++ b/apps/explorer/lib/explorer/market/market.ex @@ -40,7 +40,12 @@ defmodule Explorer.Market do @doc false def bulk_insert_history(records) do - Repo.insert_all(MarketHistory, records, on_conflict: :replace_all, conflict_target: [:date]) + records_without_zeroes = + Enum.reject(records, fn item -> + Decimal.equal?(item.closing_price, 0) && Decimal.equal?(item.opening_price, 0) + end) + + Repo.insert_all(MarketHistory, records_without_zeroes, on_conflict: :replace_all, conflict_target: [:date]) end def add_price(%{symbol: symbol} = token) do diff --git a/apps/explorer/test/explorer/market/history/source/crypto_compare_test.exs b/apps/explorer/test/explorer/market/history/source/crypto_compare_test.exs index 9b662549c2..1fc4223f72 100644 --- a/apps/explorer/test/explorer/market/history/source/crypto_compare_test.exs +++ b/apps/explorer/test/explorer/market/history/source/crypto_compare_test.exs @@ -86,5 +86,59 @@ defmodule Explorer.Market.History.Source.CryptoCompareTest do assert :error == CryptoCompare.fetch_history(3) end + + test "rejects empty prices", %{bypass: bypass} do + json = """ + { + "Response": "Success", + "Type": 100, + "Aggregated": false, + "Data": [ + { + "time": 1524528000, + "close": 0, + "high": 9741.91, + "low": 8957.68, + "open": 0, + "volumefrom": 136352.05, + "volumeto": 1276464750.74 + }, + { + "time": 1524614400, + "close": 0, + "high": 9765.23, + "low": 8757.06, + "open": 0, + "volumefrom": 192797.41, + "volumeto": 1779806222.98 + }, + { + "time": 1524700800, + "close": 8804.32, + "high": 8965.84, + "low": 8669.38, + "open": 8873.57, + "volumefrom": 74704.5, + "volumeto": 661168891 + } + ], + "TimeTo": 1524700800, + "TimeFrom": 1523836800, + "FirstValueInArray": true, + "ConversionType": { + "type": "direct", + "conversionSymbol": "" + } + } + """ + + Bypass.expect(bypass, fn conn -> Conn.resp(conn, 200, json) end) + + expected = [ + %{closing_price: Decimal.from_float(8804.32), date: ~D[2018-04-26], opening_price: Decimal.from_float(8873.57)} + ] + + assert {:ok, expected} == CryptoCompare.fetch_history(3) + end end end diff --git a/apps/explorer/test/explorer/market/market_test.exs b/apps/explorer/test/explorer/market/market_test.exs index 7e399ea4cd..584f1a35be 100644 --- a/apps/explorer/test/explorer/market/market_test.exs +++ b/apps/explorer/test/explorer/market/market_test.exs @@ -72,6 +72,25 @@ defmodule Explorer.MarketTest do assert missing_records == %{} end + test "doesn't replace existing records with zeros" do + date = ~D[2018-04-01] + + {:ok, old_record} = + Repo.insert(%MarketHistory{date: date, closing_price: Decimal.new(1), opening_price: Decimal.new(1)}) + + new_record = %{ + date: date, + closing_price: Decimal.new(0), + opening_price: Decimal.new(0) + } + + Market.bulk_insert_history([new_record]) + + fetched_record = Repo.get_by(MarketHistory, date: date) + assert fetched_record.closing_price == old_record.closing_price + assert fetched_record.opening_price == old_record.opening_price + end + test "overrides existing records on date conflict" do date = ~D[2018-04-01] Repo.insert(%MarketHistory{date: date}) From 96009699af5b0c2ee633846fe2fabbdc833487ac Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Mon, 8 Jul 2019 10:46:22 +0300 Subject: [PATCH 2/3] add CHANGELOG entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d4e6f755f..2b4440658c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ - [#2294](https://github.com/poanetwork/blockscout/pull/2294) - add healthy block period checking endpoint ### Fixes +- [#2311](https://github.com/poanetwork/blockscout/pull/2311) - fix market history overriding with zeroes - [#2299](https://github.com/poanetwork/blockscout/pull/2299) - fix interpolation in error message - [#2303](https://github.com/poanetwork/blockscout/pull/2303) - fix transaction csv download link - [#2304](https://github.com/poanetwork/blockscout/pull/2304) - footer grid fix for md resolution From 92f0447c2db559f90df34c17273c9c5e7e297de0 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Wed, 10 Jul 2019 14:04:59 +0300 Subject: [PATCH 3/3] move rejection logic to private function --- .../market/history/source/crypto_compare.ex | 32 +++++++++++-------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/apps/explorer/lib/explorer/market/history/source/crypto_compare.ex b/apps/explorer/lib/explorer/market/history/source/crypto_compare.ex index 09959f6cca..6888d83b2b 100644 --- a/apps/explorer/lib/explorer/market/history/source/crypto_compare.ex +++ b/apps/explorer/lib/explorer/market/history/source/crypto_compare.ex @@ -24,7 +24,12 @@ defmodule Explorer.Market.History.Source.CryptoCompare do case HTTPoison.get(url, headers) do {:ok, %Response{body: body, status_code: 200}} -> - {:ok, format_data(body)} + result = + body + |> format_data() + |> reject_zeros() + + {:ok, result} _ -> :error @@ -48,18 +53,13 @@ defmodule Explorer.Market.History.Source.CryptoCompare do defp format_data(data) do json = Jason.decode!(data) - items = - for item <- json["Data"] do - %{ - closing_price: Decimal.new(to_string(item["close"])), - date: date(item["time"]), - opening_price: Decimal.new(to_string(item["open"])) - } - end - - Enum.reject(items, fn item -> - Decimal.equal?(item.closing_price, 0) && Decimal.equal?(item.opening_price, 0) - end) + for item <- json["Data"] do + %{ + closing_price: Decimal.new(to_string(item["close"])), + date: date(item["time"]), + opening_price: Decimal.new(to_string(item["open"])) + } + end end @spec history_url(non_neg_integer()) :: String.t() @@ -72,4 +72,10 @@ defmodule Explorer.Market.History.Source.CryptoCompare do "#{base_url()}/data/histoday?#{URI.encode_query(query_params)}" end + + defp reject_zeros(items) do + Enum.reject(items, fn item -> + Decimal.equal?(item.closing_price, 0) && Decimal.equal?(item.opening_price, 0) + end) + end end