From 8e1bfc058919fc17e608cc2632fdd1fc9c8d04fe Mon Sep 17 00:00:00 2001
From: Luke Imhoff <luke.imhoff@dockyard.com>
Date: Tue, 18 Dec 2018 09:32:32 -0600
Subject: [PATCH] Convert DBConnection.ConnectionError tcp recv: closed to
 {:error, :timeout}

Resolves #1251
---
 apps/explorer/lib/explorer/chain/import.ex    |  8 +++++-
 apps/explorer/lib/explorer/repo.ex            |  5 +++-
 .../lib/indexer/block/catchup/fetcher.ex      | 13 +++++++---
 .../lib/indexer/block/realtime/fetcher.ex     | 26 ++++++++++++-------
 .../lib/indexer/block/uncle/fetcher.ex        | 11 ++++++++
 5 files changed, 47 insertions(+), 16 deletions(-)

diff --git a/apps/explorer/lib/explorer/chain/import.ex b/apps/explorer/lib/explorer/chain/import.ex
index 43b5a66f89..ac9295fca3 100644
--- a/apps/explorer/lib/explorer/chain/import.ex
+++ b/apps/explorer/lib/explorer/chain/import.ex
@@ -53,7 +53,7 @@ defmodule Explorer.Chain.Import do
 
   @type all_result ::
           {:ok, %{unquote_splicing(quoted_runner_imported)}}
-          | {:error, [Changeset.t()]}
+          | {:error, [Changeset.t()] | :timeout}
           | {:error, step :: Ecto.Multi.name(), failed_value :: any(),
              changes_so_far :: %{optional(Ecto.Multi.name()) => any()}}
 
@@ -334,6 +334,12 @@ defmodule Explorer.Chain.Import do
         {:error, _, _, _} = error -> {:halt, error}
       end
     end)
+  rescue
+    exception in DBConnection.ConnectionError ->
+      case Exception.message(exception) do
+        "tcp recv: closed" <> _ -> {:error, :timeout}
+        _ -> reraise exception, __STACKTRACE__
+      end
   end
 
   defp import_transaction(multi, options) when is_map(options) do
diff --git a/apps/explorer/lib/explorer/repo.ex b/apps/explorer/lib/explorer/repo.ex
index 9584987a4a..1bd5adda1e 100644
--- a/apps/explorer/lib/explorer/repo.ex
+++ b/apps/explorer/lib/explorer/repo.ex
@@ -54,6 +54,9 @@ defmodule Explorer.Repo do
                 to_string(kind),
                 " using options because of error.\n",
                 "\n",
+                "Chunk Size: ",
+                chunk |> length() |> to_string(),
+                "\n",
                 "Chunk:\n",
                 "\n",
                 inspect(chunk, limit: :infinity, printable_limit: :infinity),
@@ -66,7 +69,7 @@ defmodule Explorer.Repo do
                 "\n",
                 "Exception:\n",
                 "\n",
-                Exception.format(:error, exception)
+                Exception.format(:error, exception, __STACKTRACE__)
               ]
             end)
 
diff --git a/apps/indexer/lib/indexer/block/catchup/fetcher.ex b/apps/indexer/lib/indexer/block/catchup/fetcher.ex
index 6c3b2866cb..ba51d8450d 100644
--- a/apps/indexer/lib/indexer/block/catchup/fetcher.ex
+++ b/apps/indexer/lib/indexer/block/catchup/fetcher.ex
@@ -187,10 +187,15 @@ defmodule Indexer.Block.Catchup.Fetcher do
 
         {:ok, inserted: inserted}
 
-      {:error, {:import, [%Changeset{} | _] = changesets}} = error ->
-        Logger.error(fn ->
-          ["failed to validate: ", inspect(changesets), ". Retrying."]
-        end)
+      {:error, {:import = step, [%Changeset{} | _] = changesets}} = error ->
+        Logger.error(fn -> ["failed to validate: ", inspect(changesets), ". Retrying."] end, step: step)
+
+        push_back(sequence, range)
+
+        error
+
+      {:error, {:import = step, reason}} = error ->
+        Logger.error(fn -> [inspect(reason), ". Retrying."] end, step: step)
 
         push_back(sequence, range)
 
diff --git a/apps/indexer/lib/indexer/block/realtime/fetcher.ex b/apps/indexer/lib/indexer/block/realtime/fetcher.ex
index b320e3485c..f9a6d83cc3 100644
--- a/apps/indexer/lib/indexer/block/realtime/fetcher.ex
+++ b/apps/indexer/lib/indexer/block/realtime/fetcher.ex
@@ -198,7 +198,7 @@ defmodule Indexer.Block.Realtime.Fetcher do
           ]
         end)
 
-      {:error, {:import, [%Changeset{} | _] = changesets}} ->
+      {:error, {:import = step, [%Changeset{} | _] = changesets}} ->
         params = %{
           changesets: changesets,
           block_number_to_fetch: block_number_to_fetch,
@@ -207,17 +207,23 @@ defmodule Indexer.Block.Realtime.Fetcher do
         }
 
         if retry_fetch_and_import_block(params) == :ignore do
-          Logger.error(fn ->
-            [
-              "failed to validate for block ",
-              to_string(block_number_to_fetch),
-              ": ",
-              inspect(changesets),
-              ".  Block will be retried by catchup indexer."
-            ]
-          end)
+          Logger.error(
+            fn ->
+              [
+                "failed to validate for block ",
+                to_string(block_number_to_fetch),
+                ": ",
+                inspect(changesets),
+                ".  Block will be retried by catchup indexer."
+              ]
+            end,
+            step: step
+          )
         end
 
+      {:error, {:import = step, reason}} ->
+        Logger.error(fn -> inspect(reason) end, step: step)
+
       {:error, {step, reason}} ->
         Logger.error(
           fn ->
diff --git a/apps/indexer/lib/indexer/block/uncle/fetcher.ex b/apps/indexer/lib/indexer/block/uncle/fetcher.ex
index abd9f88e6b..efc216181e 100644
--- a/apps/indexer/lib/indexer/block/uncle/fetcher.ex
+++ b/apps/indexer/lib/indexer/block/uncle/fetcher.ex
@@ -8,6 +8,7 @@ defmodule Indexer.Block.Uncle.Fetcher do
 
   require Logger
 
+  alias Ecto.Changeset
   alias EthereumJSONRPC.Blocks
   alias Explorer.Chain
   alias Explorer.Chain.Hash
@@ -118,6 +119,16 @@ defmodule Indexer.Block.Uncle.Fetcher do
       {:ok, _} ->
         retry(errors)
 
+      {:error, {:import = step, [%Changeset{} | _] = changesets}} ->
+        Logger.error(fn -> ["Failed to validate: ", inspect(changesets)] end, step: step)
+
+        {:retry, original_entries}
+
+      {:error, {:import = step, reason}} ->
+        Logger.error(fn -> inspect(reason) end, step: step)
+
+        {:retry, original_entries}
+
       {:error, step, failed_value, _changes_so_far} ->
         Logger.error(fn -> ["failed to import: ", inspect(failed_value)] end,
           step: step,