Merge branch 'master' into ab-ipc-client

CHANGELOG.md

@@ -2,10 +2,13 @@
 
 ### Features
 - [#2791](https://github.com/poanetwork/blockscout/pull/2791) - add ipc client
+- [#2449](https://github.com/poanetwork/blockscout/pull/2449) - add ability to send notification events through postgres notify
 
 ### Fixes
 
 ### Chore
+- [#2817](https://github.com/poanetwork/blockscout/pull/2817) - move docker integration documentation to blockscout docs
+- [#2801](https://github.com/poanetwork/blockscout/pull/2801) - remove unused clause in address_to_unique_tokens query
 
 
 ## 2.1.0-beta

apps/explorer/config/config.exs

@@ -16,6 +16,11 @@ config :explorer,
   include_uncles_in_average_block_time:
     if(System.get_env("UNCLES_IN_AVERAGE_BLOCK_TIME") == "true", do: true, else: false),
   healthy_blocks_period: System.get_env("HEALTHY_BLOCKS_PERIOD") || :timer.minutes(5),
+  realtime_events_sender:
+    if(System.get_env("DISABLE_WEBAPP") != "true",
+      do: Explorer.Chain.Events.SimpleSender,
+      else: Explorer.Chain.Events.DBSender
+    ),
   index_internal_transactions_for_token_transfers:
     if(System.get_env("INTERNAL_TRANSACTIONOS_FOR_TOKEN_TRANSFERS") == "true", do: true, else: false)
 
@@ -29,6 +34,13 @@ config :explorer, Explorer.Counters.AverageBlockTime,
   enabled: true,
   period: average_block_period
 
+config :explorer, Explorer.Chain.Events.Listener,
+  enabled:
+    if(System.get_env("DISABLE_WEBAPP") == nil && System.get_env("DISABLE_INDEXER") == nil,
+      do: false,
+      else: true
+    )
+
 config :explorer, Explorer.ChainSpec.GenesisData,
   enabled: true,
   chain_spec_path: System.get_env("CHAIN_SPEC_PATH"),

apps/explorer/config/test.exs

@@ -43,6 +43,9 @@ end
 config :explorer, Explorer.ExchangeRates.Source.TransactionAndLog,
   secondary_source: Explorer.ExchangeRates.Source.OneCoinSource
 
+config :explorer,
+  realtime_events_sender: Explorer.Chain.Events.SimpleSender
+
 variant =
   if is_nil(System.get_env("ETHEREUM_JSONRPC_VARIANT")) do
     "parity"

apps/explorer/lib/explorer/application.ex

@@ -71,6 +71,7 @@ defmodule Explorer.Application do
       configure(Explorer.ChainSpec.GenesisData),
       configure(Explorer.KnownTokens),
       configure(Explorer.Market.History.Cataloger),
+      configure(Explorer.Chain.Events.Listener),
       configure(Explorer.Counters.AddressesWithBalanceCounter),
       configure(Explorer.Counters.AddressesCounter),
       configure(Explorer.Counters.AverageBlockTime),

apps/explorer/lib/explorer/chain/events/db_sender.ex

@@ -0,0 +1,26 @@
+defmodule Explorer.Chain.Events.DBSender do
+  @moduledoc """
+  Sends events to Postgres.
+  """
+  alias Explorer.Repo
+
+  def send_data(event_type) do
+    payload = encode_payload({:chain_event, event_type})
+    send_notify(payload)
+  end
+
+  def send_data(event_type, broadcast_type, event_data) do
+    payload = encode_payload({:chain_event, event_type, broadcast_type, event_data})
+    send_notify(payload)
+  end
+
+  defp encode_payload(payload) do
+    payload
+    |> :erlang.term_to_binary([:compressed])
+    |> Base.encode64()
+  end
+
+  defp send_notify(payload) do
+    Repo.query!("select pg_notify('chain_event', $1::text);", [payload])
+  end
+end

apps/explorer/lib/explorer/chain/events/listener.ex

@@ -0,0 +1,55 @@
+defmodule Explorer.Chain.Events.Listener do
+  @moduledoc """
+  Listens and publishes events from PG
+  """
+
+  use GenServer
+
+  alias Postgrex.Notifications
+
+  def start_link(_) do
+    GenServer.start_link(__MODULE__, "chain_event", name: __MODULE__)
+  end
+
+  def init(channel) do
+    {:ok, pid} =
+      :explorer
+      |> Application.get_env(Explorer.Repo)
+      |> Notifications.start_link()
+
+    ref = Notifications.listen!(pid, channel)
+
+    {:ok, {pid, ref, channel}}
+  end
+
+  def handle_info({:notification, _pid, _ref, _topic, payload}, state) do
+    payload
+    |> decode_payload!()
+    |> broadcast()
+
+    {:noreply, state}
+  end
+
+  # sobelow_skip ["Misc.BinToTerm"]
+  defp decode_payload!(payload) do
+    payload
+    |> Base.decode64!()
+    |> :erlang.binary_to_term()
+  end
+
+  defp broadcast({:chain_event, event_type} = event) do
+    Registry.dispatch(Registry.ChainEvents, event_type, fn entries ->
+      for {pid, _registered_val} <- entries do
+        send(pid, event)
+      end
+    end)
+  end
+
+  defp broadcast({:chain_event, event_type, broadcast_type, _data} = event) do
+    Registry.dispatch(Registry.ChainEvents, {event_type, broadcast_type}, fn entries ->
+      for {pid, _registered_val} <- entries do
+        send(pid, event)
+      end
+    end)
+  end
+end

apps/explorer/lib/explorer/chain/events/publisher.ex

@@ -16,14 +16,15 @@ defmodule Explorer.Chain.Events.Publisher do
   @spec broadcast(atom()) :: :ok
   def broadcast(event_type) do
     send_data(event_type)
+    :ok
   end
 
   defp send_data(event_type) do
-    Registry.dispatch(Registry.ChainEvents, event_type, fn entries ->
+    sender().send_data(event_type)
-      for {pid, _registered_val} <- entries do
-        send(pid, {:chain_event, event_type})
   end
-    end)
+
+  defp sender do
+    Application.get_env(:explorer, :realtime_events_sender)
   end
 
   # The :catchup type of event is not being consumed right now.
@@ -32,10 +33,6 @@ defmodule Explorer.Chain.Events.Publisher do
   defp send_data(_event_type, :catchup, _event_data), do: :ok
 
   defp send_data(event_type, broadcast_type, event_data) do
-    Registry.dispatch(Registry.ChainEvents, {event_type, broadcast_type}, fn entries ->
+    sender().send_data(event_type, broadcast_type, event_data)
-      for {pid, _registered_val} <- entries do
-        send(pid, {:chain_event, event_type, broadcast_type, event_data})
-      end
-    end)
   end
 end

apps/explorer/lib/explorer/chain/events/simple_sender.ex

@@ -0,0 +1,21 @@
+defmodule Explorer.Chain.Events.SimpleSender do
+  @moduledoc """
+  Publishes events through Registry without intermediate levels.
+  """
+
+  def send_data(event_type, broadcast_type, event_data) do
+    Registry.dispatch(Registry.ChainEvents, {event_type, broadcast_type}, fn entries ->
+      for {pid, _registered_val} <- entries do
+        send(pid, {:chain_event, event_type, broadcast_type, event_data})
+      end
+    end)
+  end
+
+  def send_data(event_type) do
+    Registry.dispatch(Registry.ChainEvents, event_type, fn entries ->
+      for {pid, _registered_val} <- entries do
+        send(pid, {:chain_event, event_type})
+      end
+    end)
+  end
+end

apps/explorer/lib/explorer/chain/token_transfer.ex

@@ -280,7 +280,7 @@ defmodule Explorer.Chain.TokenTransfer do
       tt in TokenTransfer,
       left_join: instance in Instance,
       on: tt.token_contract_address_hash == instance.token_contract_address_hash and tt.token_id == instance.token_id,
-      where: tt.token_contract_address_hash == ^contract_address_hash and tt.token_id == tt.token_id,
+      where: tt.token_contract_address_hash == ^contract_address_hash,
       order_by: [desc: tt.block_number],
       distinct: tt.token_id,
       preload: [:to_address],

apps/explorer/test/explorer/chain/events/publisher_test.exs

@@ -15,7 +15,7 @@ defmodule Explorer.Chain.Events.PublisherTest do
 
       Publisher.broadcast([{event_type, event_data}], broadcast_type)
 
-      assert_received {:chain_event, ^event_type, ^broadcast_type, []}
+      assert_receive {:chain_event, ^event_type, ^broadcast_type, []}
     end
 
     test "won't send chain_event of catchup type" do
@@ -59,7 +59,7 @@ defmodule Explorer.Chain.Events.PublisherTest do
 
       Publisher.broadcast(event_type)
 
-      assert_received {:chain_event, ^event_type}
+      assert_receive {:chain_event, ^event_type}
     end
   end
 end

apps/explorer/test/explorer/chain/events/subscriber_test.exs

@@ -15,7 +15,7 @@ defmodule Explorer.Chain.Events.SubscriberTest do
 
       Publisher.broadcast([{event_type, event_data}], broadcast_type)
 
-      assert_received {:chain_event, :blocks, :realtime, []}
+      assert_receive {:chain_event, :blocks, :realtime, []}
     end
   end
 
@@ -27,7 +27,7 @@ defmodule Explorer.Chain.Events.SubscriberTest do
 
       Publisher.broadcast(event_type)
 
-      assert_received {:chain_event, :exchange_rate}
+      assert_receive {:chain_event, :exchange_rate}
     end
   end
 end

apps/explorer/test/explorer/chain/import_test.exs

@@ -470,27 +470,27 @@ defmodule Explorer.Chain.ImportTest do
     test "publishes addresses with updated fetched_coin_balance data to subscribers on insert" do
       Subscriber.to(:addresses, :realtime)
       Import.all(@import_data)
-      assert_received {:chain_event, :addresses, :realtime, [%Address{}, %Address{}, %Address{}]}
+      assert_receive {:chain_event, :addresses, :realtime, [%Address{}, %Address{}, %Address{}]}
     end
 
     test "publishes block data to subscribers on insert" do
       Subscriber.to(:blocks, :realtime)
       Import.all(@import_data)
-      assert_received {:chain_event, :blocks, :realtime, [%Block{}]}
+      assert_receive {:chain_event, :blocks, :realtime, [%Block{}]}
     end
 
     test "publishes internal_transaction data to subscribers on insert" do
       Subscriber.to(:internal_transactions, :realtime)
       Import.all(@import_data)
 
-      assert_received {:chain_event, :internal_transactions, :realtime,
+      assert_receive {:chain_event, :internal_transactions, :realtime,
                       [%{transaction_hash: _, index: _}, %{transaction_hash: _, index: _}]}
     end
 
     test "publishes transactions data to subscribers on insert" do
       Subscriber.to(:transactions, :realtime)
       Import.all(@import_data)
-      assert_received {:chain_event, :transactions, :realtime, [%Transaction{}]}
+      assert_receive {:chain_event, :transactions, :realtime, [%Transaction{}]}
     end
 
     test "publishes token_transfers data to subscribers on insert" do
@@ -498,7 +498,7 @@ defmodule Explorer.Chain.ImportTest do
 
       Import.all(@import_data)
 
-      assert_received {:chain_event, :token_transfers, :realtime, [%TokenTransfer{}]}
+      assert_receive {:chain_event, :token_transfers, :realtime, [%TokenTransfer{}]}
     end
 
     test "does not broadcast if broadcast option is false" do

docker/README.md

@@ -1,77 +1,5 @@
-# BlockScout Docker integration
+# BlockScout Docker Integration
 
-For now this integration is not production ready. It made only for local usage only !
+This integration is not production ready, and should be used for local BlockScout deployment only.
-
-## How to use ?
-First of all, blockscout requires `PostgreSQL` server for working. 
-It will be provided by starting script (new docker image will be created named `postgres`)
-
-**Starting command**
-`make start` - will set everything up and start blockscout in container.
-To connect it to your local environment you will have to configure it using [env variables](#env-variables)
-
-Example connecting to local `ganache` instance running on port `2000` on Mac/Windows:
-```bash
-COIN=DAI \
-ETHEREUM_JSONRPC_VARIANT=ganache \ 
-ETHEREUM_JSONRPC_HTTP_URL=http://host.docker.internal:2000 \
-ETHEREUM_JSONRPC_WS_URL=ws://host.docker.internal:2000 \
-make start
-```
-
-Blockscout will be available on `localhost:4000`
-
-**Note**
-On mac/Windows Docker provides with a special URL `host.docker.internal` that will be available into container and routed to your local machine.
-On Linux docker is starting using `--network=host` and all services should be available on `localhost`
-
-### Migrations
-
-By default, `Makefile` will do migrations for you on `PostgreSQL` creation. 
-But you could run migrations manually using `make migrate` command.
-
-**WARNING** Migrations will clean up your local database !
-
-## Env variables
-
-BlockScout supports 3 different JSON RPC Variants.
-Variant could be configured using `ETHEREUM_JSONRPC_VARIANT` environment variable.
-
-Example: 
-```bash
-ETHEREUM_JSONRPC_VARIANT=ganache make start
-```
-
-Available options are:
-
- * `parity` - Parity JSON RPC (**Default one**)
- * `geth` - Geth JSON RPC
- * `ganache` - Ganache JSON RPC
- 
-
-| Variable | Description | Default value |
-| -------- | ----------- | ------------- |
-| `ETHEREUM_JSONRPC_VARIANT` | Variant of your JSON RPC service: `parity`, `geth` or `ganache` | `parity` |
-| `ETHEREUM_JSONRPC_HTTP_URL` | HTTP JSON RPC URL Only for `geth` or `ganache` variant | Different per JSONRPC variant |
-| `ETHEREUM_JSONRPC_WS_URL` | WS JSON RPC url | Different per JSONRPC variant |
-| `ETHEREUM_JSONRPC_TRACE_URL` | Trace URL **Only for `parity` variant** | `http://localhost:8545` |
-| `COIN` | Default Coin | `POA` |
-| `LOGO` | Coin logo | Empty | 
-| `NETWORK` | Network | Empty |
-| `SUBNETWORK` | Subnetwork | Empty |
-| `NETWORK_ICON` | Network icon | Empty | 
-| `NETWORK_PATH` | Network path | `/` |
-
-
-`ETHEREUM_JSONRPC_HTTP_URL` default values:
-
- * For `parity` - `http://localhost:8545`
- * For `geth` - `https://mainnet.infura.io/8lTvJTKmHPCHazkneJsY`
- * For `ganache` - `http://localhost:7545`
-
-`ETHEREUM_JSONRPC_WS_URL` default values:
-
- * For `parity` - `ws://localhost:8546`
- * For `geth` - `wss://mainnet.infura.io/8lTvJTKmHPCHazkneJsY/ws`
- * For `ganache` - `ws://localhost:7545`
 
+For usage instructions and ENV variables, see the [docker integration documentation](https://docs.blockscout.com/for-developers/information-and-settings/docker-integration-local-use-only).