From f2b10cc6b4b32f6a83d9408187e4d649148e470c Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Wed, 24 Jul 2024 12:15:03 +0300 Subject: [PATCH] chore: disable public metrics by default, set 1 day as default period of update (#10469) --- apps/explorer/lib/explorer/chain/metrics.ex | 24 +++---- .../lib/explorer/chain/metrics/queries.ex | 70 ++++++++++--------- .../lib/explorer/prometheus/instrumenter.ex | 56 +++++++-------- config/runtime.exs | 4 +- docker-compose/envs/common-blockscout.env | 3 +- 5 files changed, 82 insertions(+), 75 deletions(-) diff --git a/apps/explorer/lib/explorer/chain/metrics.ex b/apps/explorer/lib/explorer/chain/metrics.ex index 9031ed8cb5..b90e925e42 100644 --- a/apps/explorer/lib/explorer/chain/metrics.ex +++ b/apps/explorer/lib/explorer/chain/metrics.ex @@ -13,13 +13,13 @@ defmodule Explorer.Chain.Metrics do @interval :timer.hours(1) @options [timeout: 60_000, api?: true] @metrics_list [ - :weekly_success_transactions_number, - :weekly_deployed_smart_contracts_number, - :weekly_verified_smart_contracts_number, - :weekly_new_addresses_number, - :weekly_new_tokens_number, - :weekly_new_token_transfers_number, - :weekly_simplified_active_addresses_number + :success_transactions_number, + :deployed_smart_contracts_number, + :verified_smart_contracts_number, + :new_addresses_number, + :new_tokens_number, + :new_token_transfers_number, + :simplified_active_addresses_number ] @spec start_link(term()) :: GenServer.on_start() @@ -28,11 +28,11 @@ defmodule Explorer.Chain.Metrics do end def init(_) do - if Application.get_env(:explorer, __MODULE__)[:disabled?] do - :ignore - else + if Application.get_env(:explorer, __MODULE__)[:enabled] do send(self(), :set_metrics) {:ok, %{}} + else + :ignore end end @@ -69,12 +69,12 @@ defmodule Explorer.Chain.Metrics do defp set_handler_metric(metric) do func = String.to_atom(to_string(metric) <> "_query") - weekly_transactions_count = + transactions_count = Queries |> apply(func, []) |> select_repo(@options).one() - apply(Instrumenter, metric, [weekly_transactions_count]) + apply(Instrumenter, metric, [transactions_count]) end defp schedule_next_run do diff --git a/apps/explorer/lib/explorer/chain/metrics/queries.ex b/apps/explorer/lib/explorer/chain/metrics/queries.ex index 41b0c628b7..c68b829422 100644 --- a/apps/explorer/lib/explorer/chain/metrics/queries.ex +++ b/apps/explorer/lib/explorer/chain/metrics/queries.ex @@ -31,18 +31,18 @@ defmodule Explorer.Chain.Metrics.Queries do @doc """ Retrieves the query for fetching the number of successful transactions in a week. """ - @spec weekly_success_transactions_number_query() :: Ecto.Query.t() - def weekly_success_transactions_number_query do + @spec success_transactions_number_query() :: Ecto.Query.t() + def success_transactions_number_query do if DenormalizationHelper.transactions_denormalization_finished?() do Transaction - |> where([tx], tx.block_timestamp >= ago(7, "day")) + |> where([tx], tx.block_timestamp >= ago(^update_period_hours(), "hour")) |> where([tx], tx.block_consensus == true) |> where([tx], tx.status == ^1) |> select([tx], count(tx.hash)) else Transaction |> join(:inner, [tx], block in assoc(tx, :block)) - |> where([tx, block], block.timestamp >= ago(7, "day")) + |> where([tx, block], block.timestamp >= ago(^update_period_hours(), "hour")) |> where([tx, block], block.consensus == true) |> where([tx, block], tx.status == ^1) |> select([tx, block], count(tx.hash)) @@ -52,13 +52,13 @@ defmodule Explorer.Chain.Metrics.Queries do @doc """ Retrieves the query for the number of smart contracts deployed in the current week. """ - @spec weekly_deployed_smart_contracts_number_query() :: Ecto.Query.t() - def weekly_deployed_smart_contracts_number_query do + @spec deployed_smart_contracts_number_query() :: Ecto.Query.t() + def deployed_smart_contracts_number_query do transactions_query = if DenormalizationHelper.transactions_denormalization_finished?() do Transaction |> where([tx], not is_nil(tx.created_contract_address_hash)) - |> where([tx], tx.block_timestamp >= ago(7, "day")) + |> where([tx], tx.block_timestamp >= ago(^update_period_hours(), "hour")) |> where([tx], tx.block_consensus == true) |> where([tx], tx.status == ^1) |> select([tx], tx.created_contract_address_hash) @@ -67,7 +67,7 @@ defmodule Explorer.Chain.Metrics.Queries do |> join(:inner, [tx], block in assoc(tx, :block)) |> where([tx], not is_nil(tx.created_contract_address_hash)) |> where([tx, block], block.consensus == true) - |> where([tx, block], block.timestamp >= ago(7, "day")) + |> where([tx, block], block.timestamp >= ago(^update_period_hours(), "hour")) |> where([tx, block], tx.status == ^1) |> select([tx, block], tx.created_contract_address_hash) end @@ -77,7 +77,7 @@ defmodule Explorer.Chain.Metrics.Queries do # InternalTransaction # |> join(:inner, [it], transaction in assoc(it, :transaction)) # |> where([it, tx], not is_nil(it.created_contract_address_hash)) - # |> where([it, tx], tx.block_timestamp >= ago(7, "day")) + # |> where([it, tx], tx.block_timestamp >= ago(^update_period_hours(), "hour")) # |> where([it, tx], tx.block_consensus == true) # |> where([it, tx], tx.status == ^1) # |> select([it, tx], it.created_contract_address_hash) @@ -97,41 +97,41 @@ defmodule Explorer.Chain.Metrics.Queries do @doc """ Retrieves the query for the number of verified smart contracts in the current week. """ - @spec weekly_verified_smart_contracts_number_query() :: Ecto.Query.t() - def weekly_verified_smart_contracts_number_query do + @spec verified_smart_contracts_number_query() :: Ecto.Query.t() + def verified_smart_contracts_number_query do SmartContract - |> where([sc], sc.inserted_at >= ago(7, "day")) + |> where([sc], sc.inserted_at >= ago(^update_period_hours(), "hour")) |> select([sc], count(sc.address_hash)) end @doc """ Retrieves the query for the number of new addresses in the current week. """ - @spec weekly_new_addresses_number_query() :: Ecto.Query.t() - def weekly_new_addresses_number_query do + @spec new_addresses_number_query() :: Ecto.Query.t() + def new_addresses_number_query do Address - |> where([a], a.inserted_at >= ago(7, "day")) + |> where([a], a.inserted_at >= ago(^update_period_hours(), "hour")) |> select([a], count(a.hash)) end @doc """ Retrieves the query for the number of new tokens detected in the current week. """ - @spec weekly_new_tokens_number_query() :: Ecto.Query.t() - def weekly_new_tokens_number_query do + @spec new_tokens_number_query() :: Ecto.Query.t() + def new_tokens_number_query do Token - |> where([token], token.inserted_at >= ago(7, "day")) + |> where([token], token.inserted_at >= ago(^update_period_hours(), "hour")) |> select([token], count(token.contract_address_hash)) end @doc """ Retrieves the query for the number of new token transfers detected in the current week. """ - @spec weekly_new_token_transfers_number_query() :: Ecto.Query.t() - def weekly_new_token_transfers_number_query do + @spec new_token_transfers_number_query() :: Ecto.Query.t() + def new_token_transfers_number_query do TokenTransfer |> join(:inner, [tt], block in Block, on: block.number == tt.block_number) - |> where([tt, block], block.timestamp >= ago(7, "day")) + |> where([tt, block], block.timestamp >= ago(^update_period_hours(), "hour")) |> where([tt, block], block.consensus == true) |> select([tt, block], fragment("COUNT(*)")) end @@ -139,17 +139,17 @@ defmodule Explorer.Chain.Metrics.Queries do @doc """ Retrieves the query for the number of addresses initiated transactions in the current week. """ - @spec weekly_simplified_active_addresses_number_query() :: Ecto.Query.t() - def weekly_simplified_active_addresses_number_query do + @spec simplified_active_addresses_number_query() :: Ecto.Query.t() + def simplified_active_addresses_number_query do if DenormalizationHelper.transactions_denormalization_finished?() do Transaction - |> where([tx], tx.block_timestamp >= ago(7, "day")) + |> where([tx], tx.block_timestamp >= ago(^update_period_hours(), "hour")) |> where([tx], tx.block_consensus == true) |> select([tx], fragment("COUNT(DISTINCT(?))", tx.from_address_hash)) else Transaction |> join(:inner, [tx], block in assoc(tx, :block)) - |> where([tx, block], block.timestamp >= ago(7, "day")) + |> where([tx, block], block.timestamp >= ago(^update_period_hours(), "hour")) |> where([tx, block], block.consensus == true) |> select([tx], fragment("COUNT(DISTINCT(?))", tx.from_address_hash)) end @@ -159,12 +159,12 @@ defmodule Explorer.Chain.Metrics.Queries do Retrieves the query for the number of active EOA and smart-contract addresses (from/to/contract participated in transactions, internal transactions, token transfers) in the current week. This query is currently unused since the very low performance: it doesn't return results in 1 hour. """ - @spec weekly_active_addresses_number_query() :: Ecto.Query.t() - def weekly_active_addresses_number_query do + @spec active_addresses_number_query() :: Ecto.Query.t() + def active_addresses_number_query do transactions_query = if DenormalizationHelper.transactions_denormalization_finished?() do Transaction - |> where([tx], tx.block_timestamp >= ago(7, "day")) + |> where([tx], tx.block_timestamp >= ago(^update_period_hours(), "hour")) |> where([tx], tx.block_consensus == true) |> distinct(true) |> select([tx], %{ @@ -179,7 +179,7 @@ defmodule Explorer.Chain.Metrics.Queries do else Transaction |> join(:inner, [tx], block in assoc(tx, :block)) - |> where([tx, block], block.timestamp >= ago(7, "day")) + |> where([tx, block], block.timestamp >= ago(^update_period_hours(), "hour")) |> where([tx, block], block.consensus == true) |> distinct(true) |> select([tx, block], %{ @@ -197,7 +197,7 @@ defmodule Explorer.Chain.Metrics.Queries do if DenormalizationHelper.transactions_denormalization_finished?() do InternalTransaction |> join(:inner, [it], transaction in assoc(it, :transaction)) - |> where([it, tx], tx.block_timestamp >= ago(7, "day")) + |> where([it, tx], tx.block_timestamp >= ago(^update_period_hours(), "hour")) |> where([it, tx], tx.block_consensus == true) |> where([it, tx], tx.status == ^1) |> select([it, tx], %{ @@ -214,7 +214,7 @@ defmodule Explorer.Chain.Metrics.Queries do InternalTransaction |> join(:inner, [it], transaction in assoc(it, :transaction)) |> join(:inner, [tx], block in assoc(tx, :block)) - |> where([it, tx, block], tx.block_timestamp >= ago(7, "day")) + |> where([it, tx, block], tx.block_timestamp >= ago(^update_period_hours(), "hour")) |> where([it, tx, block], block.consensus == true) |> where([it, tx, block], tx.status == ^1) |> select([it, tx, block], %{ @@ -233,7 +233,7 @@ defmodule Explorer.Chain.Metrics.Queries do if DenormalizationHelper.transactions_denormalization_finished?() do TokenTransfer |> join(:inner, [tt], transaction in assoc(tt, :transaction)) - |> where([tt, tx], tx.block_timestamp >= ago(7, "day")) + |> where([tt, tx], tx.block_timestamp >= ago(^update_period_hours(), "hour")) |> where([tt, tx], tx.block_consensus == true) |> where([tt, tx], tx.status == ^1) |> select([tt, tx], %{ @@ -245,7 +245,7 @@ defmodule Explorer.Chain.Metrics.Queries do TokenTransfer |> join(:inner, [tt], transaction in assoc(tt, :transaction)) |> join(:inner, [tx], block in assoc(tx, :block)) - |> where([tt, tx, block], tx.block_timestamp >= ago(7, "day")) + |> where([tt, tx, block], tx.block_timestamp >= ago(^update_period_hours(), "hour")) |> where([tt, tx, block], block.consensus == true) |> where([tt, tx, block], tx.status == ^1) |> select([tt, tx, block], %{ @@ -266,4 +266,8 @@ defmodule Explorer.Chain.Metrics.Queries do select: fragment("COUNT(DISTINCT ?)", q.address_hash) ) end + + defp update_period_hours do + Application.get_env(:explorer, Explorer.Chain.Metrics)[:update_period_hours] + end end diff --git a/apps/explorer/lib/explorer/prometheus/instrumenter.ex b/apps/explorer/lib/explorer/prometheus/instrumenter.ex index ca3d619db9..9eed93f68f 100644 --- a/apps/explorer/lib/explorer/prometheus/instrumenter.ex +++ b/apps/explorer/lib/explorer/prometheus/instrumenter.ex @@ -14,45 +14,45 @@ defmodule Explorer.Prometheus.Instrumenter do ] @gauge [ - name: :weekly_success_transactions_number, - help: "Number of successful transactions in the last 7 days", + name: :success_transactions_number, + help: "Number of successful transactions in the period (default is 1 day)", registry: :public ] @gauge [ - name: :weekly_deployed_smart_contracts_number, + name: :deployed_smart_contracts_number, help: - "Number of deployed smart-contracts (smart-contracts from internal transactions are not accounted) in the last 7 days", + "Number of deployed smart-contracts (smart-contracts from internal transactions are not accounted) in the period (default is 1 day)", registry: :public ] @gauge [ - name: :weekly_verified_smart_contracts_number, - help: "Number of verified smart-contracts in the last 7 days", + name: :verified_smart_contracts_number, + help: "Number of verified smart-contracts in the period (default is 1 day)", registry: :public ] @gauge [ - name: :weekly_new_addresses_number, - help: "Number of new wallet addresses in the last 7 days", + name: :new_addresses_number, + help: "Number of new wallet addresses in the period (default is 1 day)", registry: :public ] @gauge [ - name: :weekly_new_tokens_number, - help: "Number of new tokens detected in the last 7 days", + name: :new_tokens_number, + help: "Number of new tokens detected in the period (default is 1 day)", registry: :public ] @gauge [ - name: :weekly_new_token_transfers_number, - help: "Number of new token transfers detected in the last 7 days", + name: :new_token_transfers_number, + help: "Number of new token transfers detected in the period (default is 1 day)", registry: :public ] @gauge [ - name: :weekly_active_addresses_number, - help: "Number of active EOA addresses (participated in transactions in to/from) in the last 7 days", + name: :active_addresses_number, + help: "Number of active EOA addresses (participated in transactions in to/from) in the period (default is 1 day)", registry: :public ] @@ -64,31 +64,31 @@ defmodule Explorer.Prometheus.Instrumenter do result end - def weekly_success_transactions_number(number) do - Gauge.set([name: :weekly_success_transactions_number, registry: :public], number) + def success_transactions_number(number) do + Gauge.set([name: :success_transactions_number, registry: :public], number) end - def weekly_deployed_smart_contracts_number(number) do - Gauge.set([name: :weekly_deployed_smart_contracts_number, registry: :public], number) + def deployed_smart_contracts_number(number) do + Gauge.set([name: :deployed_smart_contracts_number, registry: :public], number) end - def weekly_verified_smart_contracts_number(number) do - Gauge.set([name: :weekly_verified_smart_contracts_number, registry: :public], number) + def verified_smart_contracts_number(number) do + Gauge.set([name: :verified_smart_contracts_number, registry: :public], number) end - def weekly_new_addresses_number(number) do - Gauge.set([name: :weekly_new_addresses_number, registry: :public], number) + def new_addresses_number(number) do + Gauge.set([name: :new_addresses_number, registry: :public], number) end - def weekly_new_tokens_number(number) do - Gauge.set([name: :weekly_new_tokens_number, registry: :public], number) + def new_tokens_number(number) do + Gauge.set([name: :new_tokens_number, registry: :public], number) end - def weekly_new_token_transfers_number(number) do - Gauge.set([name: :weekly_new_token_transfers_number, registry: :public], number) + def new_token_transfers_number(number) do + Gauge.set([name: :new_token_transfers_number, registry: :public], number) end - def weekly_simplified_active_addresses_number(number) do - Gauge.set([name: :weekly_active_addresses_number, registry: :public], number) + def simplified_active_addresses_number(number) do + Gauge.set([name: :active_addresses_number, registry: :public], number) end end diff --git a/config/runtime.exs b/config/runtime.exs index e5de09cf0c..29feadfdd4 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -596,7 +596,9 @@ config :explorer, Explorer.Chain.TokenTransfer, whitelisted_weth_contracts: ConfigHelper.parse_list_env_var("WHITELISTED_WETH_CONTRACTS", ""), weth_token_transfers_filtering_enabled: ConfigHelper.parse_bool_env_var("WETH_TOKEN_TRANSFERS_FILTERING_ENABLED") -config :explorer, Explorer.Chain.Metrics, disabled?: ConfigHelper.parse_bool_env_var("METRICS_DISABLE_PUBLIC", "false") +config :explorer, Explorer.Chain.Metrics, + enabled: ConfigHelper.parse_bool_env_var("PUBLIC_METRICS_ENABLED", "false"), + update_period_hours: ConfigHelper.parse_integer_env_var("PUBLIC_METRICS_UPDATE_PERIOD_HOURS", 24) ############### ### Indexer ### diff --git a/docker-compose/envs/common-blockscout.env b/docker-compose/envs/common-blockscout.env index 02cccd8d4c..3f880b7089 100644 --- a/docker-compose/envs/common-blockscout.env +++ b/docker-compose/envs/common-blockscout.env @@ -398,4 +398,5 @@ TENDERLY_CHAIN_PATH= # WHITELISTED_WETH_CONTRACTS= # SANITIZE_INCORRECT_WETH_BATCH_SIZE=100 # SANITIZE_INCORRECT_WETH_CONCURRENCY=1 -# METRICS_DISABLE_PUBLIC= \ No newline at end of file +# PUBLIC_METRICS_ENABLED= +# PUBLIC_METRICS_UPDATE_PERIOD_HOURS= \ No newline at end of file